/*
GoCheese -- Python private package repository and caching proxy
-Copyright (C) 2019-2022 Sergey Matveev <stargrave@stargrave.org>
+Copyright (C) 2019-2023 Sergey Matveev <stargrave@stargrave.org>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
)
const (
+ HashAlgoBLAKE2b256 = "blake2b_256"
HashAlgoSHA256 = "sha256"
- HashAlgoBLAKE2b256 = "blake2_256"
HashAlgoSHA512 = "sha512"
HashAlgoMD5 = "md5"
- GPGSigExt = ".asc"
InternalFlag = ".internal"
)
PyPIURLParsed *url.URL
PyPIHTTPTransport http.Transport
KnownHashAlgos []string = []string{
- HashAlgoSHA256,
HashAlgoBLAKE2b256,
+ HashAlgoSHA256,
HashAlgoSHA512,
HashAlgoMD5,
}
func agentedReq(url string) *http.Request {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
req.Header.Set("User-Agent", UserAgent)
return req
w http.ResponseWriter,
r *http.Request,
pkgName, filenameGet string,
- gpgUpdate bool,
) bool {
if _, err := os.Stat(filepath.Join(Root, pkgName, InternalFlag)); err == nil {
return true
Name: MDFieldToRecField[recField],
Value: jsonField,
}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
for _, m := range []RecFieldToValuesMap{
Name: MDFieldToRecField[m.recField],
Value: v,
}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
}
Name: MDFieldToRecField[recField],
Value: jsonField,
}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
Name: MDFieldToRecField[m.recField],
Value: v,
}); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
}
if _, err = wr.WriteFieldMultiline(
MDFieldDescription, lines,
); err != nil {
- log.Fatalln(err)
+ log.Fatal(err)
}
}
}
path := filepath.Join(dirPath, MDFile)
existing, err := os.ReadFile(path)
- if err != nil || bytes.Compare(existing, buf.Bytes()) != 0 {
+ if err != nil || !bytes.Equal(existing, buf.Bytes()) {
if err = WriteFileSync(dirPath, path, buf.Bytes(), now); err != nil {
log.Println("error", r.RemoteAddr, "refresh-json", path, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
mtimes := make(map[string]time.Time)
+ digestsBLAKE2b256 := make(map[string][]byte)
+ digestsSHA256 := make(map[string][]byte)
+ digestsSHA512 := make(map[string][]byte)
for _, releases := range allReleases {
for _, rel := range releases {
if rel.Filename == "" || rel.UploadTimeISO8601 == "" {
return false
}
mtimes[rel.Filename] = t.Truncate(time.Second)
+ if d := rel.Digests[HashAlgoBLAKE2b256]; d != "" {
+ digestsBLAKE2b256[rel.Filename], err = hex.DecodeString(d)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not decode blake2b_256 digest:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ }
+ if d := rel.Digests[HashAlgoSHA256]; d != "" {
+ digestsSHA256[rel.Filename], err = hex.DecodeString(d)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not decode sha256 digest:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ }
+ if d := rel.Digests[HashAlgoSHA512]; d != "" {
+ digestsSHA512[rel.Filename], err = hex.DecodeString(d)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not decode sha512 digest:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ }
}
}
return false
}
- if pkgURL.Fragment == "" {
- log.Println(r.RemoteAddr, "pypi", filename, "no digest")
- http.Error(w, "no digest provided", http.StatusBadGateway)
- return false
- }
- digestInfo := strings.Split(pkgURL.Fragment, "=")
- if len(digestInfo) == 1 {
- // Ancient non PEP-0503 PyPIs, assume MD5
- digestInfo = []string{"md5", digestInfo[0]}
- } else if len(digestInfo) != 2 {
- log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
- http.Error(w, "invalid digest provided", http.StatusBadGateway)
- return false
- }
- digest, err := hex.DecodeString(digestInfo[1])
- if err != nil {
- log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
- http.Error(w, err.Error(), http.StatusBadGateway)
- return false
- }
- hashAlgo := digestInfo[0]
+ var hashAlgo string
var hasherNew func() hash.Hash
- var hashSize int
- switch hashAlgo {
- case HashAlgoMD5:
- hasherNew = md5.New
- hashSize = md5.Size
- case HashAlgoSHA256:
+ var digest []byte
+ if d := digestsBLAKE2b256[filename]; d != nil {
+ hasherNew = blake2b256New
+ hashAlgo = HashAlgoBLAKE2b256
+ digest = d
+ } else if d := digestsSHA256[filename]; d != nil {
hasherNew = sha256.New
- hashSize = sha256.Size
- case HashAlgoSHA512:
+ hashAlgo = HashAlgoSHA256
+ digest = d
+ } else if d := digestsSHA512[filename]; d != nil {
hasherNew = sha512.New
- hashSize = sha512.Size
- case HashAlgoBLAKE2b256:
- hasherNew = blake2b256New
- hashSize = blake2b.Size256
- default:
- log.Println(
- "error", r.RemoteAddr, "pypi",
- filename, "unknown digest", hashAlgo,
- )
- http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
- return false
- }
- if len(digest) != hashSize {
- log.Println(
- "error", r.RemoteAddr, "pypi",
- filename, "invalid digest length")
- http.Error(w, "invalid digest length", http.StatusBadGateway)
- return false
+ hashAlgo = HashAlgoSHA512
+ digest = d
+ } else {
+ if pkgURL.Fragment == "" {
+ log.Println(r.RemoteAddr, "pypi", filename, "no digest")
+ http.Error(w, "no digest provided", http.StatusBadGateway)
+ return false
+ }
+ digestInfo := strings.Split(pkgURL.Fragment, "=")
+ if len(digestInfo) == 1 {
+ // Ancient non PEP-0503 PyPIs, assume MD5
+ digestInfo = []string{"md5", digestInfo[0]}
+ } else if len(digestInfo) != 2 {
+ log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
+ http.Error(w, "invalid digest provided", http.StatusBadGateway)
+ return false
+ }
+ var err error
+ digest, err = hex.DecodeString(digestInfo[1])
+ if err != nil {
+ log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest")
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return false
+ }
+ hashAlgo = digestInfo[0]
+ var hashSize int
+ switch hashAlgo {
+ case HashAlgoBLAKE2b256:
+ hasherNew = blake2b256New
+ hashSize = blake2b.Size256
+ case HashAlgoSHA256:
+ hasherNew = sha256.New
+ hashSize = sha256.Size
+ case HashAlgoSHA512:
+ hasherNew = sha512.New
+ hashSize = sha512.Size
+ case HashAlgoMD5:
+ hasherNew = md5.New
+ hashSize = md5.Size
+ default:
+ log.Println(
+ "error", r.RemoteAddr, "pypi",
+ filename, "unknown digest", hashAlgo,
+ )
+ http.Error(w, "unknown digest algorithm", http.StatusBadGateway)
+ return false
+ }
+ if len(digest) != hashSize {
+ log.Println(
+ "error", r.RemoteAddr, "pypi",
+ filename, "invalid digest length")
+ http.Error(w, "invalid digest length", http.StatusBadGateway)
+ return false
+ }
}
pkgURL.Fragment = ""
return false
}
hasher := hasherNew()
- hasherSHA256 := sha256.New()
hasherBLAKE2b256 := blake2b256New()
+ hasherSHA256 := sha256.New()
dst, err := TempFile(dirPath)
if err != nil {
log.Println("error", r.RemoteAddr, "pypi", filename, err)
}
dstBuf := bufio.NewWriter(dst)
wrs := []io.Writer{hasher, dstBuf}
- if hashAlgo != HashAlgoSHA256 {
- wrs = append(wrs, hasherSHA256)
- }
if hashAlgo != HashAlgoBLAKE2b256 {
wrs = append(wrs, hasherBLAKE2b256)
}
+ if hashAlgo != HashAlgoSHA256 {
+ wrs = append(wrs, hasherSHA256)
+ }
wr := io.MultiWriter(wrs...)
if _, err = io.Copy(wr, resp.Body); err != nil {
os.Remove(dst.Name())
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
- if bytes.Compare(hasher.Sum(nil), digest) != 0 {
+ if !bytes.Equal(hasher.Sum(nil), digest) {
log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
os.Remove(dst.Name())
dst.Close()
return false
}
if digestStored, err := os.ReadFile(path + "." + hashAlgo); err == nil &&
- bytes.Compare(digest, digestStored) != 0 {
+ !bytes.Equal(digest, digestStored) {
err = errors.New("stored digest mismatch")
log.Println("error", r.RemoteAddr, "pypi", filename, err)
os.Remove(dst.Name())
return false
}
- var digestSHA256 []byte
var digestBLAKE2b256 []byte
- if hashAlgo == HashAlgoSHA256 {
- digestSHA256 = hasher.Sum(nil)
- } else {
- digestSHA256 = hasherSHA256.Sum(nil)
- }
+ var digestSHA256 []byte
if hashAlgo == HashAlgoBLAKE2b256 {
digestBLAKE2b256 = hasher.Sum(nil)
} else {
digestBLAKE2b256 = hasherBLAKE2b256.Sum(nil)
}
+ if hashAlgo == HashAlgoSHA256 {
+ digestSHA256 = hasher.Sum(nil)
+ } else {
+ digestSHA256 = hasherSHA256.Sum(nil)
+ }
if err = WriteFileSync(
- dirPath, path+"."+HashAlgoSHA256,
- digestSHA256, mtime,
+ dirPath, path+"."+HashAlgoBLAKE2b256,
+ digestBLAKE2b256, mtime,
); err != nil {
log.Println(
"error", r.RemoteAddr, "pypi",
- path+"."+HashAlgoSHA256, err,
+ path+"."+HashAlgoBLAKE2b256, err,
)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
if err = WriteFileSync(
- dirPath, path+"."+HashAlgoBLAKE2b256,
- digestBLAKE2b256, mtime,
+ dirPath, path+"."+HashAlgoSHA256,
+ digestSHA256, mtime,
); err != nil {
log.Println(
"error", r.RemoteAddr, "pypi",
- path+"."+HashAlgoBLAKE2b256, err,
+ path+"."+HashAlgoSHA256, err,
)
http.Error(w, err.Error(), http.StatusInternalServerError)
return false
}
}
- if filename == filenameGet || gpgUpdate {
- resp, err := c.Do(agentedReq(uri + GPGSigExt))
- if err != nil {
- goto GPGSigSkip
- }
- if resp.StatusCode != http.StatusOK {
- resp.Body.Close()
- goto GPGSigSkip
- }
- sig, err := io.ReadAll(resp.Body)
- resp.Body.Close()
- if err != nil {
- goto GPGSigSkip
- }
- if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) {
- log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP")
- goto GPGSigSkip
- }
- if err = WriteFileSync(dirPath, path+GPGSigExt, sig, mtime); err != nil {
- log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
- return false
- }
- log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded")
- }
- if mtimeExists {
- stat, err := os.Stat(path + GPGSigExt)
- if err == nil && !stat.ModTime().Truncate(time.Second).Equal(mtime) {
- log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "touch")
- if err = os.Chtimes(path+GPGSigExt, mtime, mtime); err != nil {
- log.Println("error", r.RemoteAddr, "pypi", filename, err)
- http.Error(w, err.Error(), http.StatusInternalServerError)
- }
- }
- }
-
- GPGSigSkip:
if digest == nil {
continue
}