+ c := http.Client{Transport: &PyPIHTTPTransport}
+ dirPath := filepath.Join(Root, pkgName)
+ now := time.Now()
+
+ var allReleases map[string][]*PkgReleaseInfo
+ if *JSONURL != "" {
+ resp, err := c.Do(agentedReq(*JSONURL + pkgName + "/json"))
+ if err != nil {
+ log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err)
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return false
+ }
+ if resp.StatusCode != http.StatusOK {
+ resp.Body.Close()
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "HTTP status:", resp.Status,
+ )
+ http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
+ return false
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ resp.Body.Close()
+ var buf bytes.Buffer
+ var description string
+ wr := recfile.NewWriter(&buf)
+ var meta PkgMeta
+ err = json.Unmarshal(body, &meta)
+ if err == nil {
+ for _, m := range [][2]string{
+ {MetadataFieldName, meta.Info.Name},
+ {MetadataFieldVersion, meta.Info.Version},
+ {MetadataFieldSummary, meta.Info.Summary},
+ {MetadataFieldDescriptionContentType, meta.Info.DescriptionContentType},
+ {MetadataFieldKeywords, meta.Info.Keywords},
+ {MetadataFieldHomePage, meta.Info.HomePage},
+ {MetadataFieldAuthor, meta.Info.Author},
+ {MetadataFieldAuthorEmail, meta.Info.AuthorEmail},
+ {MetadataFieldMaintainer, meta.Info.Maintainer},
+ {MetadataFieldMaintainerEmail, meta.Info.MaintainerEmail},
+ {MetadataFieldLicense, meta.Info.License},
+ {MetadataFieldRequiresPython, meta.Info.RequiresPython},
+ } {
+ recField, jsonField := m[0], m[1]
+ if jsonField == "" {
+ continue
+ }
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(recField),
+ Value: jsonField,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ for _, m := range []RecFieldToValuesMap{
+ {MetadataFieldClassifier, meta.Info.Classifier},
+ {MetadataFieldPlatform, meta.Info.Platform},
+ {MetadataFieldSupportedPlatform, meta.Info.SupportedPlatform},
+ {MetadataFieldRequiresDist, meta.Info.RequiresDist},
+ {MetadataFieldRequiresExternal, meta.Info.RequiresExternal},
+ {MetadataFieldProjectURL, meta.Info.ProjectURL},
+ {MetadataFieldProvidesExtra, meta.Info.ProvidesExtra},
+ } {
+ for _, v := range m.jsonFields {
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(m.recField),
+ Value: v,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ }
+ description = meta.Info.Description
+ allReleases = meta.Releases
+ } else {
+ var metaStripped PkgMetaStripped
+ err = json.Unmarshal(body, &metaStripped)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not parse JSON:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ for _, m := range [][2]string{
+ {MetadataFieldName, metaStripped.Info.Name},
+ {MetadataFieldVersion, metaStripped.Info.Version},
+ {MetadataFieldSummary, metaStripped.Info.Summary},
+ {MetadataFieldDescriptionContentType, metaStripped.Info.DescriptionContentType},
+ {MetadataFieldKeywords, metaStripped.Info.Keywords},
+ {MetadataFieldHomePage, metaStripped.Info.HomePage},
+ {MetadataFieldAuthor, metaStripped.Info.Author},
+ {MetadataFieldAuthorEmail, metaStripped.Info.AuthorEmail},
+ {MetadataFieldMaintainer, metaStripped.Info.Maintainer},
+ {MetadataFieldMaintainerEmail, metaStripped.Info.MaintainerEmail},
+ {MetadataFieldLicense, metaStripped.Info.License},
+ {MetadataFieldRequiresPython, metaStripped.Info.RequiresPython},
+ } {
+ recField, jsonField := m[0], m[1]
+ if jsonField == "" {
+ continue
+ }
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(recField),
+ Value: jsonField,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+
+ for _, m := range []RecFieldToValuesMap{
+ {MetadataFieldClassifier, metaStripped.Info.Classifier},
+ {MetadataFieldRequiresDist, metaStripped.Info.RequiresDist},
+ } {
+ for _, v := range m.jsonFields {
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(m.recField),
+ Value: v,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ }
+ description = metaStripped.Info.Description
+ allReleases = metaStripped.Releases
+ }
+ lines := strings.Split(description, "\n")
+ if len(lines) > 0 {
+ if _, err = wr.WriteFieldMultiline(
+ MetadataFieldDescription, lines,
+ ); err != nil {
+ log.Fatalln(err)
+ }
+ }
+
+ if !mkdirForPkg(w, r, pkgName) {
+ return false
+ }
+ path := filepath.Join(dirPath, MetadataFile)
+ existing, err := ioutil.ReadFile(path)
+ if err != nil || bytes.Compare(existing, buf.Bytes()) != 0 {
+ if err = WriteFileSync(dirPath, path, buf.Bytes(), now); err != nil {
+ log.Println("error", r.RemoteAddr, "refresh-json", path, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return false
+ }
+ log.Println(r.RemoteAddr, "pypi", pkgName+"/"+MetadataFile, "touch")
+ }
+ }
+ mtimes := make(map[string]time.Time)
+ for _, releases := range allReleases {
+ for _, rel := range releases {
+ if rel.Filename == "" || rel.UploadTimeISO8601 == "" {
+ continue
+ }
+ t, err := time.Parse(time.RFC3339Nano, rel.UploadTimeISO8601)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not parse upload_time:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ mtimes[rel.Filename] = t.Truncate(time.Second)
+ }
+ }
+
+ resp, err := c.Do(agentedReq(*PyPIURL + pkgName + "/"))