2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
37 "golang.org/x/crypto/blake2b"
40 var pypiHTTPTransport http.Transport
42 func blake2b256New() hash.Hash {
43 h, err := blake2b.New256(nil)
50 func agentedReq(url string) *http.Request {
51 req, err := http.NewRequest("GET", url, nil)
55 req.Header.Set("User-Agent", UserAgent)
60 w http.ResponseWriter,
62 pkgName, filenameGet string,
65 if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
68 c := http.Client{Transport: &pypiHTTPTransport}
69 resp, err := c.Get(*pypiURL + pkgName + "/")
71 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
72 http.Error(w, err.Error(), http.StatusBadGateway)
75 if resp.StatusCode != http.StatusOK {
77 log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
78 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
81 body, err := ioutil.ReadAll(resp.Body)
84 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
85 http.Error(w, err.Error(), http.StatusBadGateway)
88 if !mkdirForPkg(w, r, pkgName) {
91 dirPath := filepath.Join(*root, pkgName)
92 for _, lineRaw := range bytes.Split(body, []byte("\n")) {
93 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
94 if len(submatches) == 0 {
98 filename := submatches[2]
99 pkgURL, err := url.Parse(uri)
101 log.Println("error", r.RemoteAddr, "refresh", uri, err)
102 http.Error(w, err.Error(), http.StatusBadGateway)
106 if pkgURL.Fragment == "" {
107 log.Println(r.RemoteAddr, "pypi", filename, "no digest")
108 http.Error(w, "no digest provided", http.StatusBadGateway)
111 digestInfo := strings.Split(pkgURL.Fragment, "=")
112 if len(digestInfo) == 1 {
113 // Ancient non PEP-0503 PyPIs, assume MD5
114 digestInfo = []string{"md5", digestInfo[0]}
115 } else if len(digestInfo) != 2 {
116 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
117 http.Error(w, "invalid digest provided", http.StatusBadGateway)
120 digest, err := hex.DecodeString(digestInfo[1])
122 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
123 http.Error(w, err.Error(), http.StatusBadGateway)
126 hashAlgo := digestInfo[0]
127 var hasherNew func() hash.Hash
134 hasherNew = sha256.New
135 hashSize = sha256.Size
137 hasherNew = sha512.New
138 hashSize = sha512.Size
139 case HashAlgoBLAKE2b256:
140 hasherNew = blake2b256New
141 hashSize = blake2b.Size256
143 log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
144 http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
147 if len(digest) != hashSize {
148 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
149 http.Error(w, "invalid digest length", http.StatusBadGateway)
154 if pkgURL.Host == "" {
155 uri = pypiURLParsed.ResolveReference(pkgURL).String()
157 uri = pkgURL.String()
160 path := filepath.Join(dirPath, filename)
161 if filename == filenameGet {
163 // Skip heavy remote call, when shutting down
164 http.Error(w, "shutting down", http.StatusInternalServerError)
167 log.Println(r.RemoteAddr, "pypi", filename, "download")
168 resp, err = c.Do(agentedReq(uri))
170 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
171 http.Error(w, err.Error(), http.StatusBadGateway)
174 defer resp.Body.Close()
175 if resp.StatusCode != http.StatusOK {
177 "error", r.RemoteAddr,
178 "pypi", filename, "download",
179 "HTTP status:", resp.Status,
181 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
184 hasher := hasherNew()
185 hasherSHA256 := sha256.New()
186 dst, err := TempFile(dirPath)
188 log.Println("error", r.RemoteAddr, "pypi", filename, err)
189 http.Error(w, err.Error(), http.StatusInternalServerError)
192 dstBuf := bufio.NewWriter(dst)
193 wrs := []io.Writer{hasher, dstBuf}
194 if hashAlgo != HashAlgoSHA256 {
195 wrs = append(wrs, hasherSHA256)
197 wr := io.MultiWriter(wrs...)
198 if _, err = io.Copy(wr, resp.Body); err != nil {
199 os.Remove(dst.Name())
201 log.Println("error", r.RemoteAddr, "pypi", filename, err)
202 http.Error(w, err.Error(), http.StatusInternalServerError)
205 if err = dstBuf.Flush(); err != nil {
206 os.Remove(dst.Name())
208 log.Println("error", r.RemoteAddr, "pypi", filename, err)
209 http.Error(w, err.Error(), http.StatusInternalServerError)
212 if bytes.Compare(hasher.Sum(nil), digest) != 0 {
213 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
214 os.Remove(dst.Name())
216 http.Error(w, "digest mismatch", http.StatusBadGateway)
220 if err = dst.Sync(); err != nil {
221 os.Remove(dst.Name())
223 log.Println("error", r.RemoteAddr, "pypi", filename, err)
224 http.Error(w, err.Error(), http.StatusInternalServerError)
228 if err = dst.Close(); err != nil {
229 log.Println("error", r.RemoteAddr, "pypi", filename, err)
230 http.Error(w, err.Error(), http.StatusInternalServerError)
233 if err = os.Rename(dst.Name(), path); err != nil {
234 log.Println("error", r.RemoteAddr, "pypi", filename, err)
235 http.Error(w, err.Error(), http.StatusInternalServerError)
238 if err = DirSync(dirPath); err != nil {
239 log.Println("error", r.RemoteAddr, "pypi", filename, err)
240 http.Error(w, err.Error(), http.StatusInternalServerError)
243 if hashAlgo != HashAlgoSHA256 {
244 hashAlgo = HashAlgoSHA256
245 digest = hasherSHA256.Sum(nil)
246 for _, algo := range knownHashAlgos[1:] {
247 os.Remove(path + "." + algo)
251 if filename == filenameGet || gpgUpdate {
252 if _, err = os.Stat(path); err != nil {
255 resp, err := c.Do(agentedReq(uri + GPGSigExt))
259 if resp.StatusCode != http.StatusOK {
263 sig, err := ioutil.ReadAll(resp.Body)
268 if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
269 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
272 if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
273 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
274 http.Error(w, err.Error(), http.StatusInternalServerError)
277 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
280 path = path + "." + hashAlgo
281 _, err = os.Stat(path)
285 if !os.IsNotExist(err) {
286 log.Println("error", r.RemoteAddr, "pypi", path, err)
287 http.Error(w, err.Error(), http.StatusInternalServerError)
290 log.Println(r.RemoteAddr, "pypi", filename, "touch")
291 if err = WriteFileSync(dirPath, path, digest); err != nil {
292 log.Println("error", r.RemoteAddr, "pypi", path, err)
293 http.Error(w, err.Error(), http.StatusInternalServerError)