2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
37 "golang.org/x/crypto/blake2b"
40 func blake2b256New() hash.Hash {
41 h, err := blake2b.New256(nil)
49 w http.ResponseWriter,
55 if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
58 resp, err := http.Get(*pypiURL + dir + "/")
60 http.Error(w, err.Error(), http.StatusBadGateway)
63 body, err := ioutil.ReadAll(resp.Body)
66 http.Error(w, err.Error(), http.StatusBadGateway)
69 if !mkdirForPkg(w, r, dir) {
72 dirPath := filepath.Join(*root, dir)
73 for _, lineRaw := range bytes.Split(body, []byte("\n")) {
74 submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
75 if len(submatches) == 0 {
79 filename := submatches[2]
80 pkgURL, err := url.Parse(uri)
82 http.Error(w, err.Error(), http.StatusBadGateway)
86 if pkgURL.Fragment == "" {
87 log.Println(r.RemoteAddr, "pypi", filename, "no digest provided")
88 http.Error(w, "no digest provided", http.StatusBadGateway)
91 digestInfo := strings.Split(pkgURL.Fragment, "=")
92 if len(digestInfo) == 1 {
93 // Ancient non PEP-0503 PyPIs, assume MD5
94 digestInfo = []string{"md5", digestInfo[0]}
95 } else if len(digestInfo) != 2 {
96 log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided")
97 http.Error(w, "invalid digest provided", http.StatusBadGateway)
100 digest, err := hex.DecodeString(digestInfo[1])
102 http.Error(w, err.Error(), http.StatusBadGateway)
105 hashAlgo := digestInfo[0]
106 var hasherNew func() hash.Hash
113 hasherNew = sha256.New
114 hashSize = sha256.Size
116 hasherNew = sha512.New
117 hashSize = sha512.Size
118 case HashAlgoBLAKE2b256:
119 hasherNew = blake2b256New
120 hashSize = blake2b.Size256
123 r.RemoteAddr, "pypi", filename,
124 "unknown digest algorithm", hashAlgo,
126 http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
129 if len(digest) != hashSize {
130 log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length")
131 http.Error(w, "invalid digest length", http.StatusBadGateway)
136 if pkgURL.Host == "" {
137 uri = pypiURLParsed.ResolveReference(pkgURL).String()
139 uri = pkgURL.String()
142 path := filepath.Join(dirPath, filename)
143 if filename == filenameGet {
145 // Skip heavy remote call, when shutting down
146 http.Error(w, "shutting down", http.StatusInternalServerError)
149 log.Println(r.RemoteAddr, "pypi download", filename)
150 resp, err = http.Get(uri)
152 log.Println(r.RemoteAddr, "pypi download error:", err.Error())
153 http.Error(w, err.Error(), http.StatusBadGateway)
156 defer resp.Body.Close()
157 hasher := hasherNew()
158 hasherSHA256 := sha256.New()
159 dst, err := TempFile(dirPath)
161 http.Error(w, err.Error(), http.StatusInternalServerError)
164 dstBuf := bufio.NewWriter(dst)
165 wrs := []io.Writer{hasher, dstBuf}
166 if hashAlgo != HashAlgoSHA256 {
167 wrs = append(wrs, hasherSHA256)
169 wr := io.MultiWriter(wrs...)
170 if _, err = io.Copy(wr, resp.Body); err != nil {
171 os.Remove(dst.Name())
173 http.Error(w, err.Error(), http.StatusInternalServerError)
176 if err = dstBuf.Flush(); err != nil {
177 os.Remove(dst.Name())
179 http.Error(w, err.Error(), http.StatusInternalServerError)
182 if bytes.Compare(hasher.Sum(nil), digest) != 0 {
183 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
184 os.Remove(dst.Name())
186 http.Error(w, "digest mismatch", http.StatusBadGateway)
189 if err = dst.Sync(); err != nil {
190 os.Remove(dst.Name())
192 http.Error(w, err.Error(), http.StatusInternalServerError)
195 if err = dst.Close(); err != nil {
196 http.Error(w, err.Error(), http.StatusInternalServerError)
199 if err = os.Rename(dst.Name(), path); err != nil {
200 http.Error(w, err.Error(), http.StatusInternalServerError)
203 if err = DirSync(dirPath); err != nil {
204 http.Error(w, err.Error(), http.StatusInternalServerError)
207 if hashAlgo != HashAlgoSHA256 {
208 hashAlgo = HashAlgoSHA256
209 digest = hasherSHA256.Sum(nil)
210 for _, algo := range knownHashAlgos[1:] {
211 os.Remove(path + "." + algo)
215 if filename == filenameGet || gpgUpdate {
216 if _, err = os.Stat(path); err != nil {
219 resp, err := http.Get(uri + GPGSigExt)
223 if resp.StatusCode != http.StatusOK {
227 sig, err := ioutil.ReadAll(resp.Body)
232 if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
233 log.Println(r.RemoteAddr, "pypi non PGP signature", filename)
236 if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
237 http.Error(w, err.Error(), http.StatusInternalServerError)
240 log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
243 path = path + "." + hashAlgo
244 _, err = os.Stat(path)
248 if !os.IsNotExist(err) {
249 http.Error(w, err.Error(), http.StatusInternalServerError)
252 log.Println(r.RemoteAddr, "pypi touch", filename)
253 if err = WriteFileSync(dirPath, path, digest); err != nil {
254 http.Error(w, err.Error(), http.StatusInternalServerError)