diff options
| -rw-r--r-- | app/tasks/importtasks.py | 92 | ||||
| -rw-r--r-- | app/templates/admin/list.html | 3 | ||||
| -rw-r--r-- | app/views/admin.py | 5 |
3 files changed, 97 insertions, 3 deletions
diff --git a/app/tasks/importtasks.py b/app/tasks/importtasks.py index 022f9b3..7ccd36c 100644 --- a/app/tasks/importtasks.py +++ b/app/tasks/importtasks.py @@ -215,7 +215,6 @@ def getMeta(urlstr, author): if info is not None: result["forumId"] = info.get("topicId") - print(result) return result @@ -291,3 +290,94 @@ def importRepoScreenshot(id): print("screenshot.png does not exist") return None + + + +def getDepends(package): + url = urlparse(package.repo) + urlmaker = None + if url.netloc == "github.com": + urlmaker = GithubURLMaker(url) + else: + raise TaskError("Unsupported repo") + + result = {} + if urlmaker.isValid(): + # + # Try getting depends on mod.conf + # + try: + contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8") + conf = parseConf(contents) + for key in ["depends", "optional_depends"]: + try: + result[key] = conf[key] + except KeyError: + pass + + except HTTPError: + print("mod.conf does not exist") + + if "depends" in result or "optional_depends" in result: + return result + + + # + # Try depends.txt + # + import re + pattern = re.compile("^([a-z0-9_]+)\??$") + try: + contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8") + soft = [] + hard = [] + for line in contents.split("\n"): + line = line.strip() + if pattern.match(line): + if line[len(line) - 1] == "?": + soft.append( line[:-1]) + else: + hard.append(line) + + result["depends"] = ",".join(hard) + result["optional_depends"] = ",".join(soft) + except HTTPError: + print("depends.txt does not exist") + + return result + + else: + print(TaskError("non-github depends detector not implemented yet!")) + return {} + + +def importDependencies(package, mpackage_cache): + if Dependency.query.filter_by(depender=package).count() != 0: + return + + result = getDepends(package) + + if "depends" in result: + deps = Dependency.SpecToList(package, result["depends"], mpackage_cache) + print("{} hard: {}".format(len(deps), result["depends"])) + for dep in deps: + dep.optional = False + db.session.add(dep) + + if "optional_depends" in result: + deps = Dependency.SpecToList(package, result["optional_depends"], mpackage_cache) + print("{} soft: {}".format(len(deps), result["optional_depends"])) + for dep in deps: + dep.optional = True + db.session.add(dep) + +@celery.task() +def importAllDependencies(): + Dependency.query.delete() + mpackage_cache = {} + packages = Package.query.filter_by(type=PackageType.MOD).all() + for i, p in enumerate(packages): + print("============= {} ({}/{}) =============".format(p.name, i, len(packages))) + importDependencies(p, mpackage_cache) + + db.session.commit() diff --git a/app/templates/admin/list.html b/app/templates/admin/list.html index 3c15fa9..284919d 100644 --- a/app/templates/admin/list.html +++ b/app/templates/admin/list.html @@ -17,8 +17,9 @@ <form method="post" action="" class="box-body"> <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" /> <select name="action"> - <option value="importusers" selected>Create users from mod list</option> + <option value="importusers">Create users from mod list</option> <option value="importscreenshots">Import screenshots from VCS</option> + <option value="importdepends" selected>Import dependencies from downloads</option> </select> <input type="submit" value="Perform" /> </form> diff --git a/app/views/admin.py b/app/views/admin.py index 2320cc6..b1cfed6 100644 --- a/app/views/admin.py +++ b/app/views/admin.py @@ -20,7 +20,7 @@ from flask_user import * from flask.ext import menu from app import app from app.models import * -from app.tasks.importtasks import importRepoScreenshot +from app.tasks.importtasks import importRepoScreenshot, importAllDependencies from app.tasks.forumtasks import importUsersFromModList from flask_wtf import FlaskForm from wtforms import * @@ -52,6 +52,9 @@ def admin_page(): package.soft_deleted = False db.session.commit() return redirect(url_for("admin_page")) + elif action == "importdepends": + task = importAllDependencies.delay() + return redirect(url_for("check_task", id=task.id, r=url_for("admin_page"))) else: flash("Unknown action: " + action, "error") |