X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=blobdiff_plain;f=refresh.go;h=092bd0125fa9ae555cb8a18b14c89ffd1440967f;hp=0764a03385df16a2b681499da4b6f6f3f5377adb;hb=60834a0713d5dcc6a9911511cb8618ce7358c824;hpb=948543909470b983c001f13fb267a64070dd20ae diff --git a/refresh.go b/refresh.go index 0764a03..092bd01 100644 --- a/refresh.go +++ b/refresh.go @@ -24,6 +24,7 @@ import ( "crypto/sha256" "crypto/sha512" "encoding/hex" + "encoding/json" "hash" "io" "io/ioutil" @@ -32,12 +33,34 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strings" + "time" + "go.cypherpunks.ru/recfile" "golang.org/x/crypto/blake2b" ) -var pypiHTTPTransport http.Transport +const ( + HashAlgoSHA256 = "sha256" + HashAlgoBLAKE2b256 = "blake2_256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" + GPGSigExt = ".asc" + InternalFlag = ".internal" +) + +var ( + PkgPyPI = regexp.MustCompile(`^.*]*>(.+).*$`) + PyPIURLParsed *url.URL + PyPIHTTPTransport http.Transport + KnownHashAlgos []string = []string{ + HashAlgoSHA256, + HashAlgoBLAKE2b256, + HashAlgoSHA512, + HashAlgoMD5, + } +) func blake2b256New() hash.Hash { h, err := blake2b.New256(nil) @@ -62,11 +85,177 @@ func refreshDir( pkgName, filenameGet string, gpgUpdate bool, ) bool { - if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil { + if _, err := os.Stat(filepath.Join(*Root, pkgName, InternalFlag)); err == nil { return true } - c := http.Client{Transport: &pypiHTTPTransport} - resp, err := c.Get(*pypiURL + pkgName + "/") + c := http.Client{Transport: &PyPIHTTPTransport} + dirPath := filepath.Join(*Root, pkgName) + now := time.Now() + + var allReleases map[string][]*PkgReleaseInfo + if *JSONURL != "" { + resp, err := c.Do(agentedReq(*JSONURL + pkgName + "/json")) + if err != nil { + log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err) + http.Error(w, err.Error(), http.StatusBadGateway) + return false + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + log.Println( + "error", r.RemoteAddr, "refresh-json", pkgName, + "HTTP status:", resp.Status, + ) + http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway) + return false + } + body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + var buf bytes.Buffer + var description string + wr := recfile.NewWriter(&buf) + var meta PkgMeta + err = json.Unmarshal(body, &meta) + if err == nil { + for recField, jsonField := range map[string]string{ + MetadataFieldName: meta.Info.Name, + MetadataFieldVersion: meta.Info.Version, + MetadataFieldSummary: meta.Info.Summary, + MetadataFieldDescriptionContentType: meta.Info.DescriptionContentType, + MetadataFieldKeywords: meta.Info.Keywords, + MetadataFieldHomePage: meta.Info.HomePage, + MetadataFieldAuthor: meta.Info.Author, + MetadataFieldAuthorEmail: meta.Info.AuthorEmail, + MetadataFieldMaintainer: meta.Info.Maintainer, + MetadataFieldMaintainerEmail: meta.Info.MaintainerEmail, + MetadataFieldLicense: meta.Info.License, + MetadataFieldRequiresPython: meta.Info.RequiresPython, + } { + if jsonField == "" { + continue + } + if _, err = wr.WriteFields(recfile.Field{ + Name: metadataFieldToRecField(recField), + Value: jsonField, + }); err != nil { + log.Fatalln(err) + } + } + for recField, jsonFields := range map[string][]string{ + MetadataFieldClassifier: meta.Info.Classifier, + MetadataFieldPlatform: meta.Info.Platform, + MetadataFieldSupportedPlatform: meta.Info.SupportedPlatform, + MetadataFieldRequiresDist: meta.Info.RequiresDist, + MetadataFieldRequiresExternal: meta.Info.RequiresExternal, + MetadataFieldProjectURL: meta.Info.ProjectURL, + MetadataFieldProvidesExtra: meta.Info.ProvidesExtra, + } { + for _, v := range jsonFields { + if _, err = wr.WriteFields(recfile.Field{ + Name: metadataFieldToRecField(recField), + Value: v, + }); err != nil { + log.Fatalln(err) + } + } + } + description = meta.Info.Description + allReleases = meta.Releases + } else { + var metaStripped PkgMetaStripped + err = json.Unmarshal(body, &metaStripped) + if err != nil { + log.Println( + "error", r.RemoteAddr, "refresh-json", pkgName, + "can not parse JSON:", err, + ) + http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) + return false + } + for recField, jsonField := range map[string]string{ + MetadataFieldName: metaStripped.Info.Name, + MetadataFieldVersion: metaStripped.Info.Version, + MetadataFieldSummary: metaStripped.Info.Summary, + MetadataFieldDescriptionContentType: metaStripped.Info.DescriptionContentType, + MetadataFieldKeywords: metaStripped.Info.Keywords, + MetadataFieldHomePage: metaStripped.Info.HomePage, + MetadataFieldAuthor: metaStripped.Info.Author, + MetadataFieldAuthorEmail: metaStripped.Info.AuthorEmail, + MetadataFieldMaintainer: metaStripped.Info.Maintainer, + MetadataFieldMaintainerEmail: metaStripped.Info.MaintainerEmail, + MetadataFieldLicense: metaStripped.Info.License, + MetadataFieldRequiresPython: metaStripped.Info.RequiresPython, + } { + if jsonField == "" { + continue + } + if _, err = wr.WriteFields(recfile.Field{ + Name: metadataFieldToRecField(recField), + Value: jsonField, + }); err != nil { + log.Fatalln(err) + } + } + + for recField, jsonFields := range map[string][]string{ + MetadataFieldClassifier: metaStripped.Info.Classifier, + MetadataFieldRequiresDist: metaStripped.Info.RequiresDist, + } { + for _, v := range jsonFields { + if _, err = wr.WriteFields(recfile.Field{ + Name: metadataFieldToRecField(recField), + Value: v, + }); err != nil { + log.Fatalln(err) + } + } + } + description = metaStripped.Info.Description + allReleases = metaStripped.Releases + } + lines := strings.Split(description, "\n") + if len(lines) > 0 { + if _, err = wr.WriteFieldMultiline( + MetadataFieldDescription, lines, + ); err != nil { + log.Fatalln(err) + } + } + + if !mkdirForPkg(w, r, pkgName) { + return false + } + path := filepath.Join(dirPath, MetadataFile) + existing, err := ioutil.ReadFile(path) + if err != nil || bytes.Compare(existing, buf.Bytes()) != 0 { + if err = WriteFileSync(dirPath, path, buf.Bytes(), now); err != nil { + log.Println("error", r.RemoteAddr, "refresh-json", path, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } + log.Println(r.RemoteAddr, "pypi", pkgName+"."+MetadataFile, "touch") + } + } + mtimes := make(map[string]time.Time) + for _, releases := range allReleases { + for _, rel := range releases { + if rel.Filename == "" || rel.UploadTimeISO8601 == "" { + continue + } + t, err := time.Parse(time.RFC3339Nano, rel.UploadTimeISO8601) + if err != nil { + log.Println( + "error", r.RemoteAddr, "refresh-json", pkgName, + "can not parse upload_time:", err, + ) + http.Error(w, "can not parse metadata JSON", http.StatusBadGateway) + return false + } + mtimes[rel.Filename] = t.Truncate(time.Second) + } + } + + resp, err := c.Do(agentedReq(*PyPIURL + pkgName + "/")) if err != nil { log.Println("error", r.RemoteAddr, "refresh", pkgName, err) http.Error(w, err.Error(), http.StatusBadGateway) @@ -74,7 +263,10 @@ func refreshDir( } if resp.StatusCode != http.StatusOK { resp.Body.Close() - log.Println("error", r.RemoteAddr, "refresh", pkgName, "HTTP status:", resp.Status) + log.Println( + "error", r.RemoteAddr, "refresh", pkgName, + "HTTP status:", resp.Status, + ) http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway) return false } @@ -88,9 +280,8 @@ func refreshDir( if !mkdirForPkg(w, r, pkgName) { return false } - dirPath := filepath.Join(*root, pkgName) for _, lineRaw := range bytes.Split(body, []byte("\n")) { - submatches := pkgPyPI.FindStringSubmatch(string(lineRaw)) + submatches := PkgPyPI.FindStringSubmatch(string(lineRaw)) if len(submatches) == 0 { continue } @@ -152,14 +343,18 @@ func refreshDir( pkgURL.Fragment = "" if pkgURL.Host == "" { - uri = pypiURLParsed.ResolveReference(pkgURL).String() + uri = PyPIURLParsed.ResolveReference(pkgURL).String() } else { uri = pkgURL.String() } + mtime, mtimeExists := mtimes[filename] + if !mtimeExists { + mtime = now + } path := filepath.Join(dirPath, filename) if filename == filenameGet { - if killed { + if Killed { // Skip heavy remote call, when shutting down http.Error(w, "shutting down", http.StatusInternalServerError) return false @@ -230,6 +425,10 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } + if err = os.Chtimes(dst.Name(), mtime, mtime); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } if err = os.Rename(dst.Name(), path); err != nil { log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -243,11 +442,22 @@ func refreshDir( if hashAlgo != HashAlgoSHA256 { hashAlgo = HashAlgoSHA256 digest = hasherSHA256.Sum(nil) - for _, algo := range knownHashAlgos[1:] { + for _, algo := range KnownHashAlgos[1:] { os.Remove(path + "." + algo) } } } + if mtimeExists { + stat, err := os.Stat(path) + if err == nil && !stat.ModTime().Truncate(time.Second).Equal(mtime) { + log.Println(r.RemoteAddr, "pypi", filename, "touch") + if err = os.Chtimes(path, mtime, mtime); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + } + } + if filename == filenameGet || gpgUpdate { if _, err = os.Stat(path); err != nil { goto GPGSigSkip @@ -269,26 +479,38 @@ func refreshDir( log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP") goto GPGSigSkip } - if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { + if err = WriteFileSync(dirPath, path+GPGSigExt, sig, mtime); err != nil { log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded") } + if mtimeExists { + stat, err := os.Stat(path + GPGSigExt) + if err == nil && !stat.ModTime().Truncate(time.Second).Equal(mtime) { + log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "touch") + if err = os.Chtimes(path+GPGSigExt, mtime, mtime); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + } + } + GPGSigSkip: path = path + "." + hashAlgo - _, err = os.Stat(path) - if err == nil { + stat, err := os.Stat(path) + if err == nil && + (mtimeExists && stat.ModTime().Truncate(time.Second).Equal(mtime)) { continue } - if !os.IsNotExist(err) { + if err != nil && !os.IsNotExist(err) { log.Println("error", r.RemoteAddr, "pypi", path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } log.Println(r.RemoteAddr, "pypi", filename, "touch") - if err = WriteFileSync(dirPath, path, digest); err != nil { + if err = WriteFileSync(dirPath, path, digest, mtime); err != nil { log.Println("error", r.RemoteAddr, "pypi", path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false