X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=blobdiff_plain;f=refresh.go;h=0138e7ddfbf6f93e29e13f318ee025064ac6b640;hp=092bd0125fa9ae555cb8a18b14c89ffd1440967f;hb=fe95091cb996ebc6c92bceace37b7e70c7528b67;hpb=60834a0713d5dcc6a9911511cb8618ce7358c824 diff --git a/refresh.go b/refresh.go index 092bd01..0138e7d 100644 --- a/refresh.go +++ b/refresh.go @@ -25,6 +25,7 @@ import ( "crypto/sha512" "encoding/hex" "encoding/json" + "errors" "hash" "io" "io/ioutil" @@ -79,17 +80,22 @@ func agentedReq(url string) *http.Request { return req } +type RecFieldToValuesMap struct { + recField string + jsonFields []string +} + func refreshDir( w http.ResponseWriter, r *http.Request, pkgName, filenameGet string, gpgUpdate bool, ) bool { - if _, err := os.Stat(filepath.Join(*Root, pkgName, InternalFlag)); err == nil { + if _, err := os.Stat(filepath.Join(Root, pkgName, InternalFlag)); err == nil { return true } c := http.Client{Transport: &PyPIHTTPTransport} - dirPath := filepath.Join(*Root, pkgName) + dirPath := filepath.Join(Root, pkgName) now := time.Now() var allReleases map[string][]*PkgReleaseInfo @@ -117,20 +123,21 @@ func refreshDir( var meta PkgMeta err = json.Unmarshal(body, &meta) if err == nil { - for recField, jsonField := range map[string]string{ - MetadataFieldName: meta.Info.Name, - MetadataFieldVersion: meta.Info.Version, - MetadataFieldSummary: meta.Info.Summary, - MetadataFieldDescriptionContentType: meta.Info.DescriptionContentType, - MetadataFieldKeywords: meta.Info.Keywords, - MetadataFieldHomePage: meta.Info.HomePage, - MetadataFieldAuthor: meta.Info.Author, - MetadataFieldAuthorEmail: meta.Info.AuthorEmail, - MetadataFieldMaintainer: meta.Info.Maintainer, - MetadataFieldMaintainerEmail: meta.Info.MaintainerEmail, - MetadataFieldLicense: meta.Info.License, - MetadataFieldRequiresPython: meta.Info.RequiresPython, + for _, m := range [][2]string{ + {MetadataFieldName, meta.Info.Name}, + {MetadataFieldVersion, meta.Info.Version}, + {MetadataFieldSummary, meta.Info.Summary}, + {MetadataFieldDescriptionContentType, meta.Info.DescriptionContentType}, + {MetadataFieldKeywords, meta.Info.Keywords}, + {MetadataFieldHomePage, meta.Info.HomePage}, + {MetadataFieldAuthor, meta.Info.Author}, + {MetadataFieldAuthorEmail, meta.Info.AuthorEmail}, + {MetadataFieldMaintainer, meta.Info.Maintainer}, + {MetadataFieldMaintainerEmail, meta.Info.MaintainerEmail}, + {MetadataFieldLicense, meta.Info.License}, + {MetadataFieldRequiresPython, meta.Info.RequiresPython}, } { + recField, jsonField := m[0], m[1] if jsonField == "" { continue } @@ -141,18 +148,18 @@ func refreshDir( log.Fatalln(err) } } - for recField, jsonFields := range map[string][]string{ - MetadataFieldClassifier: meta.Info.Classifier, - MetadataFieldPlatform: meta.Info.Platform, - MetadataFieldSupportedPlatform: meta.Info.SupportedPlatform, - MetadataFieldRequiresDist: meta.Info.RequiresDist, - MetadataFieldRequiresExternal: meta.Info.RequiresExternal, - MetadataFieldProjectURL: meta.Info.ProjectURL, - MetadataFieldProvidesExtra: meta.Info.ProvidesExtra, + for _, m := range []RecFieldToValuesMap{ + {MetadataFieldClassifier, meta.Info.Classifier}, + {MetadataFieldPlatform, meta.Info.Platform}, + {MetadataFieldSupportedPlatform, meta.Info.SupportedPlatform}, + {MetadataFieldRequiresDist, meta.Info.RequiresDist}, + {MetadataFieldRequiresExternal, meta.Info.RequiresExternal}, + {MetadataFieldProjectURL, meta.Info.ProjectURL}, + {MetadataFieldProvidesExtra, meta.Info.ProvidesExtra}, } { - for _, v := range jsonFields { + for _, v := range m.jsonFields { if _, err = wr.WriteFields(recfile.Field{ - Name: metadataFieldToRecField(recField), + Name: metadataFieldToRecField(m.recField), Value: v, }); err != nil { log.Fatalln(err) @@ -172,20 +179,21 @@ func refreshDir( http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) return false } - for recField, jsonField := range map[string]string{ - MetadataFieldName: metaStripped.Info.Name, - MetadataFieldVersion: metaStripped.Info.Version, - MetadataFieldSummary: metaStripped.Info.Summary, - MetadataFieldDescriptionContentType: metaStripped.Info.DescriptionContentType, - MetadataFieldKeywords: metaStripped.Info.Keywords, - MetadataFieldHomePage: metaStripped.Info.HomePage, - MetadataFieldAuthor: metaStripped.Info.Author, - MetadataFieldAuthorEmail: metaStripped.Info.AuthorEmail, - MetadataFieldMaintainer: metaStripped.Info.Maintainer, - MetadataFieldMaintainerEmail: metaStripped.Info.MaintainerEmail, - MetadataFieldLicense: metaStripped.Info.License, - MetadataFieldRequiresPython: metaStripped.Info.RequiresPython, + for _, m := range [][2]string{ + {MetadataFieldName, metaStripped.Info.Name}, + {MetadataFieldVersion, metaStripped.Info.Version}, + {MetadataFieldSummary, metaStripped.Info.Summary}, + {MetadataFieldDescriptionContentType, metaStripped.Info.DescriptionContentType}, + {MetadataFieldKeywords, metaStripped.Info.Keywords}, + {MetadataFieldHomePage, metaStripped.Info.HomePage}, + {MetadataFieldAuthor, metaStripped.Info.Author}, + {MetadataFieldAuthorEmail, metaStripped.Info.AuthorEmail}, + {MetadataFieldMaintainer, metaStripped.Info.Maintainer}, + {MetadataFieldMaintainerEmail, metaStripped.Info.MaintainerEmail}, + {MetadataFieldLicense, metaStripped.Info.License}, + {MetadataFieldRequiresPython, metaStripped.Info.RequiresPython}, } { + recField, jsonField := m[0], m[1] if jsonField == "" { continue } @@ -197,13 +205,13 @@ func refreshDir( } } - for recField, jsonFields := range map[string][]string{ - MetadataFieldClassifier: metaStripped.Info.Classifier, - MetadataFieldRequiresDist: metaStripped.Info.RequiresDist, + for _, m := range []RecFieldToValuesMap{ + {MetadataFieldClassifier, metaStripped.Info.Classifier}, + {MetadataFieldRequiresDist, metaStripped.Info.RequiresDist}, } { - for _, v := range jsonFields { + for _, v := range m.jsonFields { if _, err = wr.WriteFields(recfile.Field{ - Name: metadataFieldToRecField(recField), + Name: metadataFieldToRecField(m.recField), Value: v, }); err != nil { log.Fatalln(err) @@ -233,7 +241,7 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "pypi", pkgName+"."+MetadataFile, "touch") + log.Println(r.RemoteAddr, "pypi", pkgName+"/"+MetadataFile, "touch") } } mtimes := make(map[string]time.Time) @@ -331,12 +339,17 @@ func refreshDir( hasherNew = blake2b256New hashSize = blake2b.Size256 default: - log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo) + log.Println( + "error", r.RemoteAddr, "pypi", + filename, "unknown digest", hashAlgo, + ) http.Error(w, "unknown digest algorithm", http.StatusBadGateway) return false } if len(digest) != hashSize { - log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length") + log.Println( + "error", r.RemoteAddr, "pypi", + filename, "invalid digest length") http.Error(w, "invalid digest length", http.StatusBadGateway) return false } @@ -378,6 +391,7 @@ func refreshDir( } hasher := hasherNew() hasherSHA256 := sha256.New() + hasherBLAKE2b256 := blake2b256New() dst, err := TempFile(dirPath) if err != nil { log.Println("error", r.RemoteAddr, "pypi", filename, err) @@ -389,6 +403,9 @@ func refreshDir( if hashAlgo != HashAlgoSHA256 { wrs = append(wrs, hasherSHA256) } + if hashAlgo != HashAlgoBLAKE2b256 { + wrs = append(wrs, hasherBLAKE2b256) + } wr := io.MultiWriter(wrs...) if _, err = io.Copy(wr, resp.Body); err != nil { os.Remove(dst.Name()) @@ -411,6 +428,15 @@ func refreshDir( http.Error(w, "digest mismatch", http.StatusBadGateway) return false } + if digestStored, err := ioutil.ReadFile(path + "." + hashAlgo); err == nil && + bytes.Compare(digest, digestStored) != 0 { + err = errors.New("stored digest mismatch") + log.Println("error", r.RemoteAddr, "pypi", filename, err) + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if !NoSync { if err = dst.Sync(); err != nil { os.Remove(dst.Name()) @@ -439,13 +465,45 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } - if hashAlgo != HashAlgoSHA256 { - hashAlgo = HashAlgoSHA256 - digest = hasherSHA256.Sum(nil) - for _, algo := range KnownHashAlgos[1:] { - os.Remove(path + "." + algo) - } + + var digestSHA256 []byte + var digestBLAKE2b256 []byte + if hashAlgo == HashAlgoSHA256 { + digestSHA256 = hasher.Sum(nil) + } else { + digestSHA256 = hasherSHA256.Sum(nil) + } + if hashAlgo == HashAlgoBLAKE2b256 { + digestBLAKE2b256 = hasher.Sum(nil) + } else { + digestBLAKE2b256 = hasherBLAKE2b256.Sum(nil) } + if err = WriteFileSync( + dirPath, path+"."+HashAlgoSHA256, + digestSHA256, mtime, + ); err != nil { + log.Println( + "error", r.RemoteAddr, "pypi", + path+"."+HashAlgoSHA256, err, + ) + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + if err = WriteFileSync( + dirPath, path+"."+HashAlgoBLAKE2b256, + digestBLAKE2b256, mtime, + ); err != nil { + log.Println( + "error", r.RemoteAddr, "pypi", + path+"."+HashAlgoBLAKE2b256, err, + ) + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + for _, algo := range KnownHashAlgos[2:] { + os.Remove(path + "." + algo) + } + digest = nil } if mtimeExists { stat, err := os.Stat(path) @@ -498,10 +556,13 @@ func refreshDir( } GPGSigSkip: + if digest == nil { + continue + } path = path + "." + hashAlgo stat, err := os.Stat(path) - if err == nil && - (mtimeExists && stat.ModTime().Truncate(time.Second).Equal(mtime)) { + if err == nil && (!mtimeExists || + (mtimeExists && stat.ModTime().Truncate(time.Second).Equal(mtime))) { continue } if err != nil && !os.IsNotExist(err) {