From fb5cba4cc82efa5cae4e78c5ad75575b615de378 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:03:54 +0100 Subject: Add dependency detection to importer --- app/tasks/importtasks.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'app/tasks/importtasks.py') diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index db992b3..022f9b3 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -55,6 +55,9 @@ class GithubURLMaker: def getDescURL(self): return self.baseUrl + "/description.txt" + def getDependsURL(self): + return self.baseUrl + "/depends.txt" + def getScreenshotURL(self): return self.baseUrl + "/screenshot.png" @@ -161,7 +164,7 @@ def getMeta(urlstr, author): try: contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") conf = parseConf(contents) - for key in ["name", "description", "title"]: + for key in ["name", "description", "title", "depends", "optional_depends"]: try: result[key] = conf[key] except KeyError: @@ -179,16 +182,40 @@ def getMeta(urlstr, author): except HTTPError: print("description.txt does not exist!") + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + if not "depends" in result and not "optional_depends" in result: + try: + contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8") + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = ",".join(hard) + result["optional_depends"] = ",".join(soft) + + + except HTTPError: + print("depends.txt does not exist!") + if "description" in result: desc = result["description"] idx = desc.find(".") + 1 cutIdx = min(len(desc), 200 if idx < 5 else idx) result["short_description"] = desc[:cutIdx] + info = findModInfo(author, result.get("name"), result["repo"]) if info is not None: result["forumId"] = info.get("topicId") + print(result) return result -- cgit v1.2.3 From 746cf7f4b5bdddd4728bfb7b79abf6edc43d1cc4 Mon Sep 17 00:00:00 2001 From: rubenwardy Date: Sun, 27 May 2018 22:34:24 +0100 Subject: Add bulk dependency importer from Github --- app/tasks/importtasks.py | 92 ++++++++++++++++++++++++++++++++++++++++++- app/templates/admin/list.html | 3 +- app/views/admin.py | 5 ++- 3 files changed, 97 insertions(+), 3 deletions(-) (limited to 'app/tasks/importtasks.py') diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index 022f9b3..7ccd36c 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -215,7 +215,6 @@ def getMeta(urlstr, author): if info is not None: result["forumId"] = info.get("topicId") - print(result) return result @@ -291,3 +290,94 @@ def importRepoScreenshot(id): print("screenshot.png does not exist") return None + + + +def getDepends(package): + url = urlparse(package.repo) + urlmaker = None + if url.netloc == "github.com": + urlmaker = GithubURLMaker(url) + else: + raise TaskError("Unsupported repo") + + result = {} + if urlmaker.isValid(): + # + # Try getting depends on mod.conf + # + try: + contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") + conf = parseConf(contents) + for key in ["depends", "optional_depends"]: + try: + result[key] = conf[key] + except KeyError: + pass + + except HTTPError: + print("mod.conf does not exist") + + if "depends" in result or "optional_depends" in result: + return result + + + # + # Try depends.txt + # + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + try: + contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8") + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = ",".join(hard) + result["optional_depends"] = ",".join(soft) + except HTTPError: + print("depends.txt does not exist") + + return result + + else: + print(TaskError("non-github depends detector not implemented yet!")) + return {} + + +def importDependencies(package, mpackage_cache): + if Dependency.query.filter_by(depender=package).count() != 0: + return + + result = getDepends(package) + + if "depends" in result: + deps = Dependency.SpecToList(package, result["depends"], mpackage_cache) + print("{} hard: {}".format(len(deps), result["depends"])) + for dep in deps: + dep.optional = False + db.session.add(dep) + + if "optional_depends" in result: + deps = Dependency.SpecToList(package, result["optional_depends"], mpackage_cache) + print("{} soft: {}".format(len(deps), result["optional_depends"])) + for dep in deps: + dep.optional = True + db.session.add(dep) + +@celery.task() +def importAllDependencies(): + Dependency.query.delete() + mpackage_cache = {} + packages = Package.query.filter_by(type=PackageType.MOD).all() + for i, p in enumerate(packages): + print("============= {} ({}/{}) =============".format(p.name, i, len(packages))) + importDependencies(p, mpackage_cache) + + db.session.commit() diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index 3c15fa9..284919d 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -17,8 +17,9 @@
diff --git a/app/views/admin.py b/app/views/admin.py index 2320cc6..b1cfed6 100644 --- a/app/views/admin.py +++ b/app/views/admin.py @@ -20,7 +20,7 @@ from flask_user import * from flask.ext import menu from app import app from app.models import * -from app.tasks.importtasks import importRepoScreenshot +from app.tasks.importtasks import importRepoScreenshot, importAllDependencies from app.tasks.forumtasks import importUsersFromModList from flask_wtf import FlaskForm from wtforms import * @@ -52,6 +52,9 @@ def admin_page(): package.soft_deleted = False db.session.commit() return redirect(url_for("admin_page")) + elif action == "importdepends": + task = importAllDependencies.delay() + return redirect(url_for("check_task", id=task.id, r=url_for("admin_page"))) else: flash("Unknown action: " + action, "error") -- cgit v1.2.3