1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
import flask, json
from flask.ext.sqlalchemy import SQLAlchemy
import urllib.request
from urllib.parse import urlparse, quote_plus
from app import app
from app.models import *
from app.tasks import celery, TaskError
class GithubURLMaker:
def __init__(self, url):
# Rewrite path
import re
m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
if m is None:
return
user = m.group(1)
repo = m.group(2)
self.baseUrl = "https://raw.githubusercontent.com/{}/{}/master" \
.format(user, repo.replace(".git", ""))
self.user = user
self.repo = repo
def isValid(self):
return self.baseUrl is not None
def getRepoURL(self):
return "https://github.com/{}/{}".format(self.user, self.repo)
def getIssueTrackerURL(self):
return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
def getModConfURL(self):
return self.baseUrl + "/mod.conf"
def getDescURL(self):
return self.baseUrl + "/description.txt"
def getScreenshotURL(self):
return self.baseUrl + "/placeholder.png"
def getCommitsURL(self, branch):
return "https://api.github.com/repos/{}/{}/commits?sha={}" \
.format(self.user, self.repo, urllib.parse.quote_plus(branch))
def getCommitDownload(self, commit):
return "https://github.com/{}/{}/archive/{}.zip" \
.format(self.user, self.repo, commit)
krock_list_cache = None
krock_list_cache_by_name = None
def getKrockList():
global krock_list_cache
global krock_list_cache_by_name
if krock_list_cache is None:
contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
list = json.loads(contents)
def h(x):
if not ("title" in x and "author" in x and \
"topicId" in x and "link" in x and x["link"] != ""):
return False
import re
m = re.search("\[([A-Za-z0-9_]+)\]", x["title"])
if m is None:
return False
x["name"] = m.group(1)
return True
def g(x):
return {
"title": x["title"],
"author": x["author"],
"name": x["name"],
"topicId": x["topicId"],
"link": x["link"],
}
krock_list_cache = [g(x) for x in list if h(x)]
krock_list_cache_by_name = {}
for x in krock_list_cache:
if not x["name"] in krock_list_cache_by_name:
krock_list_cache_by_name[x["name"]] = []
krock_list_cache_by_name[x["name"]].append(x)
return krock_list_cache, krock_list_cache_by_name
def findModInfo(author, name, link):
list, lookup = getKrockList()
if name is not None and name in lookup:
if len(lookup[name]) == 1:
return lookup[name][0]
for x in lookup[name]:
if x["author"] == author:
return x
if link is not None and len(link) > 15:
for x in list:
if link in x["link"]:
return x
return None
def parseConf(string):
retval = {}
for line in string.split("\n"):
idx = line.find("=")
if idx > 0:
key = line[:idx].strip()
value = line[idx+1:].strip()
retval[key] = value
return retval
@celery.task()
def getMeta(urlstr, author):
url = urlparse(urlstr)
urlmaker = None
if url.netloc == "github.com":
urlmaker = GithubURLMaker(url)
else:
raise TaskError("Unsupported repo")
if not urlmaker.isValid():
raise TaskError("Error! Url maker not valid")
result = {}
result["repo"] = urlmaker.getRepoURL()
result["issueTracker"] = urlmaker.getIssueTrackerURL()
try:
contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
conf = parseConf(contents)
for key in ["name", "description", "title"]:
try:
result[key] = conf[key]
except KeyError:
pass
except OSError:
print("mod.conf does not exist")
if "name" in result:
result["title"] = result["name"].replace("_", " ").title()
if not "description" in result:
try:
contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
result["description"] = contents.strip()
except OSError:
print("description.txt does not exist!")
if "description" in result:
desc = result["description"]
idx = desc.find(".") + 1
cutIdx = min(len(desc), 200 if idx < 5 else idx)
result["short_description"] = desc[:cutIdx]
info = findModInfo(author, result.get("name"), result["repo"])
if info is not None:
result["forumId"] = info.get("topicId")
return result
@celery.task()
def makeVCSRelease(id, branch):
release = PackageRelease.query.get(id)
if release is None:
raise TaskError("No such release!")
if release.package is None:
raise TaskError("No package attached to release")
url = urlparse(release.package.repo)
urlmaker = None
if url.netloc == "github.com":
urlmaker = GithubURLMaker(url)
else:
raise TaskError("Unsupported repo")
if not urlmaker.isValid():
raise TaskError("Invalid github repo URL")
contents = urllib.request.urlopen(urlmaker.getCommitsURL(branch)).read().decode("utf-8")
commits = json.loads(contents)
if len(commits) == 0:
raise TaskError("No commits found")
release.url = urlmaker.getCommitDownload(commits[0]["sha"])
release.task_id = None
db.session.commit()
return release.url
|
