/*
GoCheese -- Python private package repository and caching proxy
-Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
+Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
"golang.org/x/crypto/blake2b"
)
+var pypiHTTPTransport http.Transport
+
func blake2b256New() hash.Hash {
h, err := blake2b.New256(nil)
if err != nil {
return h
}
+func agentedReq(url string) *http.Request {
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Fatalln(err)
+ }
+ req.Header.Set("User-Agent", UserAgent)
+ return req
+}
+
func refreshDir(
w http.ResponseWriter,
r *http.Request,
- dir,
- filenameGet string,
+ pkgName, filenameGet string,
gpgUpdate bool,
) bool {
- if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
+ if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil {
return true
}
- resp, err := http.Get(*pypiURL + dir + "/")
+ c := http.Client{Transport: &pypiHTTPTransport}
+ resp, err := c.Get(*pypiURL + pkgName + "/")
if err != nil {
+ log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
http.Error(w, err.Error(), http.StatusBadGateway)
return false
}
+ if resp.StatusCode != http.StatusOK {
+ resp.Body.Close()
+ log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status)
+ http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
+ return false
+ }
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
+ log.Println("error", r.RemoteAddr, "refresh", pkgName, err)
http.Error(w, err.Error(), http.StatusBadGateway)
return false
}
- if !mkdirForPkg(w, r, dir) {
+ if !mkdirForPkg(w, r, pkgName) {
return false
}
- dirPath := filepath.Join(*root, dir)
+ dirPath := filepath.Join(*root, pkgName)
for _, lineRaw := range bytes.Split(body, []byte("\n")) {
submatches := pkgPyPI.FindStringSubmatch(string(lineRaw))
if len(submatches) == 0 {
filename := submatches[2]
pkgURL, err := url.Parse(uri)
if err != nil {
+ log.Println("error", r.RemoteAddr, "refresh", uri, err)
http.Error(w, err.Error(), http.StatusBadGateway)
return false
}
if pkgURL.Fragment == "" {
- log.Println(r.RemoteAddr, "pypi", filename, "no digest provided")
+ log.Println(r.RemoteAddr, "pypi", filename, "no digest")
http.Error(w, "no digest provided", http.StatusBadGateway)
return false
}
// Ancient non PEP-0503 PyPIs, assume MD5
digestInfo = []string{"md5", digestInfo[0]}
} else if len(digestInfo) != 2 {
- log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided")
+ log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
http.Error(w, "invalid digest provided", http.StatusBadGateway)
return false
}
digest, err := hex.DecodeString(digestInfo[1])
if err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
http.Error(w, err.Error(), http.StatusBadGateway)
return false
}
hasherNew = blake2b256New
hashSize = blake2b.Size256
default:
- log.Println(
- r.RemoteAddr, "pypi", filename,
- "unknown digest algorithm", hashAlgo,
- )
+ log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo)
http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
return false
}
if len(digest) != hashSize {
- log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length")
+ log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length")
http.Error(w, "invalid digest length", http.StatusBadGateway)
return false
}
http.Error(w, "shutting down", http.StatusInternalServerError)
return false
}
- log.Println(r.RemoteAddr, "pypi download", filename)
- resp, err = http.Get(uri)
+ log.Println(r.RemoteAddr, "pypi", filename, "download")
+ resp, err = c.Do(agentedReq(uri))
if err != nil {
- log.Println(r.RemoteAddr, "pypi download error:", err.Error())
+ log.Println("error", r.RemoteAddr, "pypi", filename, "download", err)
http.Error(w, err.Error(), http.StatusBadGateway)
return false
}
defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ log.Println(
+ "error", r.RemoteAddr,
+ "pypi", filename, "download",
+ "HTTP status:", resp.Status,
+ )
+ http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
+ return false
+ }
hasher := hasherNew()
hasherSHA256 := sha256.New()
dst, err := TempFile(dirPath)
if err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if _, err = io.Copy(wr, resp.Body); err != nil {
os.Remove(dst.Name())
dst.Close()
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if err = dstBuf.Flush(); err != nil {
os.Remove(dst.Name())
dst.Close()
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
http.Error(w, "digest mismatch", http.StatusBadGateway)
return false
}
- if err = dst.Sync(); err != nil {
- os.Remove(dst.Name())
- dst.Close()
- http.Error(w, err.Error(), http.StatusInternalServerError)
- return false
+ if !NoSync {
+ if err = dst.Sync(); err != nil {
+ os.Remove(dst.Name())
+ dst.Close()
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return false
+ }
}
if err = dst.Close(); err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if err = os.Rename(dst.Name(), path); err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if err = DirSync(dirPath); err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if _, err = os.Stat(path); err != nil {
goto GPGSigSkip
}
- resp, err := http.Get(uri + GPGSigExt)
+ resp, err := c.Do(agentedReq(uri + GPGSigExt))
if err != nil {
goto GPGSigSkip
}
goto GPGSigSkip
}
if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
- log.Println(r.RemoteAddr, "pypi non PGP signature", filename)
+ log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
goto GPGSigSkip
}
if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
- log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
+ log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
}
GPGSigSkip:
path = path + "." + hashAlgo
continue
}
if !os.IsNotExist(err) {
+ log.Println("error", r.RemoteAddr, "pypi", path, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
- log.Println(r.RemoteAddr, "pypi touch", filename)
+ log.Println(r.RemoteAddr, "pypi", filename, "touch")
if err = WriteFileSync(dirPath, path, digest); err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", path, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}