diff options
author | 2024-02-21 22:15:49 +0100 | |
---|---|---|
committer | 2024-02-21 22:15:49 +0100 | |
commit | 6d8986da642b5d3b5670be40d8fe8883086c7678 (patch) | |
tree | 28e790801b6ed35f796829be66af285db4fa564d | |
parent | Use settings repository for dockerfiles (diff) | |
download | tinderbox-cluster-6d8986da642b5d3b5670be40d8fe8883086c7678.tar.gz tinderbox-cluster-6d8986da642b5d3b5670be40d8fe8883086c7678.tar.bz2 tinderbox-cluster-6d8986da642b5d3b5670be40d8fe8883086c7678.zip |
Move search pattern db to settings repository
Signed-off-by: Magnus Granberg <zorry@gentoo.org>
-rw-r--r-- | buildbot_gentoo_ci/steps/logs.py | 34 | ||||
-rw-r--r-- | py/log_parser.py | 206 |
2 files changed, 127 insertions, 113 deletions
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py index 85017a5..50e3f3f 100644 --- a/buildbot_gentoo_ci/steps/logs.py +++ b/buildbot_gentoo_ci/steps/logs.py @@ -42,10 +42,13 @@ def PersOutputOfLogParser(rc, stdout, stderr): } for s in v['pattern_info']: summary_log_dict[int(k)]['pattern_infos'].append({ - 'type' : s['type'], + 'search_type' : s['search_type'], 'status' : s['status'], - 'id' : s['id'], + 'line' : s['line'], 'search_pattern' : s['search_pattern'], + 'phase' : s['phase'], + 'uuid' : s['uuid'], + 'description' : s['description'], }) build_summery_output['summary_log_dict'] = summary_log_dict #FIXME: Handling of stderr output @@ -118,14 +121,23 @@ class SetupParserBuildLoger(BuildStep): @defer.inlineCallbacks def run(self): + self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] self.aftersteps_list = [] log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')] build_log_file_compressed = log_cpv['full_logname'] + '.xz' mastersrc_log = yield os.path.join(self.getProperty('logsdir'), build_log_file_compressed) log_py = 'log_parser.py' - config_log_py = 'logparser.json' mastersrc_py = yield os.path.join(self.master.basedir, log_py) - mastersrc_config = yield os.path.join(self.master.basedir, config_log_py) + repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(self.getProperty("project_data")['settings_repository_uuid']) + # Git clone settings repo + self.aftersteps_list.append(steps.GitLab(repourl=repository_data['url'], + name = 'RunGit', + descriptionDone=repository_data['name'], + mode=repository_data['mode'], + method=repository_data['method'], + submodules=True, + alwaysUseLatest=repository_data['alwaysuselatest'] + )) # Upload logfile to worker self.aftersteps_list.append(steps.FileDownload( mastersrc=mastersrc_log, @@ -136,11 +148,6 @@ class SetupParserBuildLoger(BuildStep): mastersrc=mastersrc_py, workerdest=log_py )) - # Upload log parser py config - self.aftersteps_list.append(steps.FileDownload( - mastersrc=mastersrc_config, - workerdest=config_log_py - )) #Untar the log shell_commad_list = [] shell_commad_list.append('xz') @@ -159,8 +166,11 @@ class SetupParserBuildLoger(BuildStep): command.append(log_cpv['full_logname']) command.append('-u') command.append(self.getProperty('project_data')['uuid']) - command.append('-d') - command.append(util.Secret("log_parser_database")) + command.append('-c') + #FIXME: set it by images/flavors + command.append('8') + #FIXME: debug + #command.append('-d') self.aftersteps_list.append(steps.SetPropertyFromCommand( name = 'RunBuildLogParser', haltOnFailure = True, @@ -211,7 +221,7 @@ class MakeIssue(BuildStep): for k, v in sorted(self.summary_log_dict.items()): # get the issue error for s in v['pattern_infos']: - if s['type'] == self.error_dict['phase'] and s['status'] == 'error': + if s['search_type'] == self.error_dict['phase'] and s['status'] == 'error': text_issue_list.append(v['text']) # add the issue error self.error_dict['cpv'] = self.getProperty('log_cpv') diff --git a/py/log_parser.py b/py/log_parser.py index eb081a9..e3725bb 100644 --- a/py/log_parser.py +++ b/py/log_parser.py @@ -7,113 +7,113 @@ import re import io import json import os -from sqlalchemy.ext.declarative import declarative_base -import sqlalchemy as sa import argparse -Base = declarative_base() - -class ProjectsPattern(Base): - __tablename__ = "projects_pattern" - id = sa.Column(sa.Integer, primary_key=True) - project_uuid = sa.Column(sa.String(36), nullable=False) - search = sa.Column(sa.String(50), nullable=False) - start = sa.Column(sa.Integer, default=0) - end = sa.Column(sa.Integer, default=0) - status = sa.Column(sa.Enum('info', 'warning', 'ignore', 'error'), default='info') - type = sa.Column(sa.Enum('info', 'qa', 'compile', 'configure', 'install', 'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack', 'ignore', 'issues', 'misc', 'elog'), default='info') - search_type = sa.Column(sa.Enum('in', 'startswith', 'endswith', 'search'), default='in') +def getJsonFromFile(path, phase): + with open(path) as f: + try: + JsonList = json.load(f) + except json.decoder.JSONDecodeError as e: + print(f"{e} in file: {path}") + return [] + return JsonList[phase] -def get_pattern_dict(project_pattern): +def get_pattern_dict(catchissue, i, uuid): + #print(catchissue) patten_dict = {} - patten_dict['id'] = project_pattern.id - patten_dict['project_uuid'] = project_pattern.project_uuid - patten_dict['search'] = project_pattern.search - patten_dict['status'] = project_pattern.status - patten_dict['type'] = project_pattern.type + patten_dict['line'] = i + patten_dict['uuid'] = uuid + patten_dict['string'] = catchissue[0] + patten_dict['start'] = catchissue[1] + patten_dict['end'] = catchissue[2] + patten_dict['status'] = catchissue[3] + patten_dict['search_type'] = catchissue[4] + patten_dict['url'] = catchissue[5] + patten_dict['description'] = catchissue[6] return patten_dict -def addPatternToList(Session, log_search_pattern, uuid): - for project_pattern in Session.query(ProjectsPattern).filter_by(project_uuid=uuid).all(): - # check if the search pattern is vaild - project_pattern_search = project_pattern.search - try: - re.compile(project_pattern_search) - except re.error: - print("Non valid regex pattern") - print(project_pattern.search) - print(project_pattern.id) +def addPatternToList(phaseList, log_search_patterns, uuid): + for phase in phaseList: + if uuid == '00000000-0000-0000-000000000000': + path = 'LogPattern' else: - if project_pattern.type == 'ignore': - log_search_pattern['ignore'].append(get_pattern_dict(project_pattern)) - if project_pattern.type == 'test': - log_search_pattern['test'].append(get_pattern_dict(project_pattern)) + path = os.path.join('Project', uuid, 'LogPattern') + CatchIssueFile = os.path.join(path, 'CatchIssues' + phase + '.json') + CatchIssueList = getJsonFromFile(CatchIssueFile, phase) + i = 3 + for catchissue in CatchIssueList: + search_pattern = get_pattern_dict(catchissue, i, uuid) + try: + re.compile(search_pattern['string']) + except re.error: + print(f"Non valid regex pattern in line: {str(search_pattern['line'])} String: {search_pattern['string']} Project: {search_pattern['uuid']} Phase: {phase}") else: - log_search_pattern['default'].append(get_pattern_dict(project_pattern)) - return log_search_pattern + log_search_patterns[phase].append(search_pattern) + i = i + 1 + return log_search_patterns -def get_log_search_pattern(Session, uuid, default_uuid): +def get_log_search_patterns(uuid): + path = os.path.join('LogPattern', 'PhaseList.json') + PhaseList = getJsonFromFile(path,'PhaseList') # get pattern from the projects and add that to log_search_pattern - log_search_pattern = {} - log_search_pattern['ignore'] = [] - log_search_pattern['default'] = [] - log_search_pattern['test'] = [] - log_search_pattern = addPatternToList(Session, log_search_pattern, uuid) - log_search_pattern = addPatternToList(Session, log_search_pattern, default_uuid) - return log_search_pattern + log_search_patterns = {} + for phase in PhaseList: + log_search_patterns[phase] = [] + uuid_default = '00000000-0000-0000-000000000000' + log_search_patterns = addPatternToList(PhaseList, log_search_patterns, uuid_default) + #log_search_pattern = addPatternToList(PhaseList, log_search_pattern, uuid) + return log_search_patterns -def get_search_pattern_match(log_search_pattern, text_line): - match_list = [] - for search_pattern in log_search_pattern: - if re.search(search_pattern['search'], text_line): - match_list.append(search_pattern) - return match_list +def get_search_pattern_match(search_pattern, text_line): + #print(f"Text: {text_line}") + if search_pattern['search_type'] == 'search': + if re.search(search_pattern['string'], text_line): + #print(f"Match string: {search_pattern['string']} Type: {search_pattern['search_type']}") + return True + elif search_pattern['search_type'] == 'startswith': + if text_line.startswith(search_pattern['string']): + #print(f"Match string: {search_pattern['string']} Type: {search_pattern['search_type']}") + return True + elif search_pattern['search_type'] == 'endswith': + if text_line.endswith(search_pattern['string']): + #print(f"Match string: {search_pattern['string']} Type: {search_pattern['search_type']}") + return True + elif search_pattern['search_type'] == 'in': + if search_pattern['string'] in text_line: + #print(f"Match string: {search_pattern['string']} Type: {search_pattern['search_type']}") + return True + else: + return False -def search_buildlog(log_search_pattern, text_line, index): +def search_buildlog(log_search_patterns, text_line, index): summary = {} - #FIXME: add check for test - # don't log ignore lines - if get_search_pattern_match(log_search_pattern['ignore'], text_line) != []: - return False - # search default pattern summary[index] = { 'text' : text_line, 'pattern_info' : [], } - search_pattern_match_list = get_search_pattern_match(log_search_pattern['default'], text_line) - if search_pattern_match_list != []: - for search_pattern_match in search_pattern_match_list: - summary[index]['pattern_info'].append({ - 'type' : search_pattern_match['type'], - 'status' : search_pattern_match['status'], - 'id' : search_pattern_match['id'], - 'search_pattern' : search_pattern_match['search'], - }) - # we add all line that start with ' * ' or '>>>' as info - if text_line.startswith(' * ') or text_line.startswith('>>>'): - summary[index]['pattern_info'].append({ - 'type' : 'info', - 'status' : 'info', - 'id' : 0, - 'search_pattern' : 'auto', - }) + for phase, search_patterns in log_search_patterns.items(): + for search_pattern in search_patterns: + match = get_search_pattern_match(search_pattern, text_line) + if phase == 'Ignore' and match: + return False + elif phase != 'Ignore' and match: + summary[index]['pattern_info'].append({ + 'search_type' : search_pattern['search_type'], + 'status' : search_pattern['status'], + 'line' : search_pattern['line'], + 'search_pattern' : search_pattern['string'], + 'phase' : phase, + 'uuid' : search_pattern['uuid'], + 'url' : search_pattern['url'], + 'description' : search_pattern['description'], + }) if summary[index]['pattern_info'] != []: + #print(f"summary: {summary}") return summary return False -def getConfigSettings(): - #configpath = os.getcwd() - with open('logparser.json') as f: - config = json.load(f) - return config - -def getDBSession(args): - engine = sa.create_engine(args.database) - Session = sa.orm.sessionmaker(bind = engine) - return Session() - -def getMultiprocessingPool(config): - return Pool(processes = int(config['core'])) +def getMultiprocessingPool(args): + return Pool(processes = int(args.cpu)) def getJsonResult(results): for r in results: @@ -128,14 +128,11 @@ def getJsonResult(results): def runLogParser(args): index = 1 logfile_text_dict = {} - config = getConfigSettings() - Session = getDBSession(args) - #mp_pool = getMultiprocessingPool(config) summary = {} #NOTE: The patten is from https://github.com/toralf/tinderbox/tree/master/data files. - # Is stored in a db instead of files. - log_search_pattern = get_log_search_pattern(Session, args.uuid, config['default_uuid']) - Session.close() + # Is stored in json files + # make dict with it + log_search_patterns = get_log_search_patterns(args.uuid) # read the log file to dict with open(args.file, encoding='utf8', errors='ignore') as f: for text_line in f: @@ -146,21 +143,28 @@ def runLogParser(args): # index = index + 1 # run the search parse pattern on the text lines #params = [(log_search_pattern, text, line_index,) for line_index, text in logfile_text_dict.items()] - with getMultiprocessingPool(config) as pool: - results = list(pool.apply_async(search_buildlog, args=(log_search_pattern, text, line_index,)) for line_index, text in logfile_text_dict.items()) - #results = pool.starmap(search_buildlog, params) - getJsonResult(results) - pool.close() - pool.join() + if not args.debug: + with getMultiprocessingPool(args) as pool: + results = list(pool.apply_async(search_buildlog, args=(log_search_patterns, text, line_index,)) for line_index, text in logfile_text_dict.items()) + getJsonResult(results) + pool.close() + pool.join() + else: + results = [] + for line_index, text in logfile_text_dict.items(): + results.append(search_buildlog(log_search_pattern, text, line_index)) + #FIXME: Json output + #getJsonResult(results) def main(): # get filename, project_uuid default_project_uuid parser = argparse.ArgumentParser() parser.add_argument("-f", "--file", required=True) parser.add_argument("-u", "--uuid", required=True) - parser.add_argument("-e", "--default-uuid", required=False) - parser.add_argument("-c", "--cpu", required=False) - parser.add_argument("-d", "--database", required=True) + parser.add_argument("-c", "--cpu", required=True) + #FIXME: add If args.debug .... wear is needed + parser.add_argument("-d", "--debug", action="store_true", required=False) + args = parser.parse_args() runLogParser(args) sys.exit() |