2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
37 "golang.org/x/crypto/blake2b"
40 var pypiHTTPTransport http.Transport
42 func blake2b256New() hash.Hash {
43 h, err := blake2b.New256(nil)
51 w http.ResponseWriter,
53 pkgName, filenameGet string,
56 if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
59 c := http.Client{Transport: &pypiHTTPTransport}
60 resp, err := c.Get(*pypiURL + pkgName + "/")
62 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
63 http.Error(w, err.Error(), http.StatusBadGateway)
66 if resp.StatusCode != http.StatusOK {
68 log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
69 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
72 body, err := ioutil.ReadAll(resp.Body)
75 log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
76 http.Error(w, err.Error(), http.StatusBadGateway)
79 if !mkdirForPkg(w, r, pkgName) {
82 dirPath := filepath.Join(*root, pkgName)
83 for _, lineRaw := range bytes.Split(body, []byte("\n")) {
84 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
85 if len(submatches) == 0 {
89 filename := submatches[2]
90 pkgURL, err := url.Parse(uri)
92 log.Println("error", r.RemoteAddr, "refresh", uri, err)
93 http.Error(w, err.Error(), http.StatusBadGateway)
97 if pkgURL.Fragment == "" {
98 log.Println(r.RemoteAddr, "pypi", filename, "no digest")
99 http.Error(w, "no digest provided", http.StatusBadGateway)
102 digestInfo := strings.Split(pkgURL.Fragment, "=")
103 if len(digestInfo) == 1 {
104 // Ancient non PEP-0503 PyPIs, assume MD5
105 digestInfo = []string{"md5", digestInfo[0]}
106 } else if len(digestInfo) != 2 {
107 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
108 http.Error(w, "invalid digest provided", http.StatusBadGateway)
111 digest, err := hex.DecodeString(digestInfo[1])
113 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
114 http.Error(w, err.Error(), http.StatusBadGateway)
117 hashAlgo := digestInfo[0]
118 var hasherNew func() hash.Hash
125 hasherNew = sha256.New
126 hashSize = sha256.Size
128 hasherNew = sha512.New
129 hashSize = sha512.Size
130 case HashAlgoBLAKE2b256:
131 hasherNew = blake2b256New
132 hashSize = blake2b.Size256
134 log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
135 http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
138 if len(digest) != hashSize {
139 log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
140 http.Error(w, "invalid digest length", http.StatusBadGateway)
145 if pkgURL.Host == "" {
146 uri = pypiURLParsed.ResolveReference(pkgURL).String()
148 uri = pkgURL.String()
151 path := filepath.Join(dirPath, filename)
152 if filename == filenameGet {
154 // Skip heavy remote call, when shutting down
155 http.Error(w, "shutting down", http.StatusInternalServerError)
158 log.Println(r.RemoteAddr, "pypi", filename, "download")
159 resp, err = http.Get(uri)
161 log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
162 http.Error(w, err.Error(), http.StatusBadGateway)
165 defer resp.Body.Close()
166 if resp.StatusCode != http.StatusOK {
168 "error", r.RemoteAddr,
169 "pypi", filename, "download",
170 "HTTP status:", resp.Status,
172 http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
175 hasher := hasherNew()
176 hasherSHA256 := sha256.New()
177 dst, err := TempFile(dirPath)
179 log.Println("error", r.RemoteAddr, "pypi", filename, err)
180 http.Error(w, err.Error(), http.StatusInternalServerError)
183 dstBuf := bufio.NewWriter(dst)
184 wrs := []io.Writer{hasher, dstBuf}
185 if hashAlgo != HashAlgoSHA256 {
186 wrs = append(wrs, hasherSHA256)
188 wr := io.MultiWriter(wrs...)
189 if _, err = io.Copy(wr, resp.Body); err != nil {
190 os.Remove(dst.Name())
192 log.Println("error", r.RemoteAddr, "pypi", filename, err)
193 http.Error(w, err.Error(), http.StatusInternalServerError)
196 if err = dstBuf.Flush(); err != nil {
197 os.Remove(dst.Name())
199 log.Println("error", r.RemoteAddr, "pypi", filename, err)
200 http.Error(w, err.Error(), http.StatusInternalServerError)
203 if bytes.Compare(hasher.Sum(nil), digest) != 0 {
204 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
205 os.Remove(dst.Name())
207 http.Error(w, "digest mismatch", http.StatusBadGateway)
210 if err = dst.Sync(); err != nil {
211 os.Remove(dst.Name())
213 log.Println("error", r.RemoteAddr, "pypi", filename, err)
214 http.Error(w, err.Error(), http.StatusInternalServerError)
217 if err = dst.Close(); err != nil {
218 log.Println("error", r.RemoteAddr, "pypi", filename, err)
219 http.Error(w, err.Error(), http.StatusInternalServerError)
222 if err = os.Rename(dst.Name(), path); err != nil {
223 log.Println("error", r.RemoteAddr, "pypi", filename, err)
224 http.Error(w, err.Error(), http.StatusInternalServerError)
227 if err = DirSync(dirPath); err != nil {
228 log.Println("error", r.RemoteAddr, "pypi", filename, err)
229 http.Error(w, err.Error(), http.StatusInternalServerError)
232 if hashAlgo != HashAlgoSHA256 {
233 hashAlgo = HashAlgoSHA256
234 digest = hasherSHA256.Sum(nil)
235 for _, algo := range knownHashAlgos[1:] {
236 os.Remove(path + "." + algo)
240 if filename == filenameGet || gpgUpdate {
241 if _, err = os.Stat(path); err != nil {
244 resp, err := http.Get(uri + GPGSigExt)
248 if resp.StatusCode != http.StatusOK {
252 sig, err := ioutil.ReadAll(resp.Body)
257 if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
258 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
261 if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
262 log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
263 http.Error(w, err.Error(), http.StatusInternalServerError)
266 log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
269 path = path + "." + hashAlgo
270 _, err = os.Stat(path)
274 if !os.IsNotExist(err) {
275 log.Println("error", r.RemoteAddr, "pypi", path, err)
276 http.Error(w, err.Error(), http.StatusInternalServerError)
279 log.Println(r.RemoteAddr, "pypi", filename, "touch")
280 if err = WriteFileSync(dirPath, path, digest); err != nil {
281 log.Println("error", r.RemoteAddr, "pypi", path, err)
282 http.Error(w, err.Error(), http.StatusInternalServerError)