+ c := http.Client{Transport: &PyPIHTTPTransport}
+ dirPath := filepath.Join(Root, pkgName)
+ now := time.Now()
+
+ var allReleases map[string][]*PkgReleaseInfo
+ if *JSONURL != "" {
+ resp, err := c.Do(agentedReq(*JSONURL + pkgName + "/json"))
+ if err != nil {
+ log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err)
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return false
+ }
+ if resp.StatusCode != http.StatusOK {
+ resp.Body.Close()
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "HTTP status:", resp.Status,
+ )
+ http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
+ return false
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ resp.Body.Close()
+ log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err)
+ http.Error(w, "can not read body", http.StatusBadGateway)
+ return false
+ }
+ resp.Body.Close()
+ var buf bytes.Buffer
+ var description string
+ wr := recfile.NewWriter(&buf)
+ var meta PkgMeta
+ err = json.Unmarshal(body, &meta)
+ if err == nil {
+ for _, m := range [][2]string{
+ {MDFieldName, meta.Info.Name},
+ {MDFieldVersion, meta.Info.Version},
+ {MDFieldSummary, meta.Info.Summary},
+ {MDFieldDescriptionContentType, meta.Info.DescriptionContentType},
+ {MDFieldKeywords, meta.Info.Keywords},
+ {MDFieldHomePage, meta.Info.HomePage},
+ {MDFieldAuthor, meta.Info.Author},
+ {MDFieldAuthorEmail, meta.Info.AuthorEmail},
+ {MDFieldMaintainer, meta.Info.Maintainer},
+ {MDFieldMaintainerEmail, meta.Info.MaintainerEmail},
+ {MDFieldLicense, meta.Info.License},
+ {MDFieldRequiresPython, meta.Info.RequiresPython},
+ } {
+ recField, jsonField := m[0], m[1]
+ if jsonField == "" {
+ continue
+ }
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: MDFieldToRecField[recField],
+ Value: jsonField,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ for _, m := range []RecFieldToValuesMap{
+ {MDFieldClassifier, meta.Info.Classifier},
+ {MDFieldPlatform, meta.Info.Platform},
+ {MDFieldSupportedPlatform, meta.Info.SupportedPlatform},
+ {MDFieldRequiresDist, meta.Info.RequiresDist},
+ {MDFieldRequiresExternal, meta.Info.RequiresExternal},
+ {MDFieldProjectURL, meta.Info.ProjectURL},
+ {MDFieldProvidesExtra, meta.Info.ProvidesExtra},
+ } {
+ for _, v := range m.jsonFields {
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: MDFieldToRecField[m.recField],
+ Value: v,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ }
+ description = meta.Info.Description
+ allReleases = meta.Releases
+ } else {
+ var metaStripped PkgMetaStripped
+ err = json.Unmarshal(body, &metaStripped)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not parse JSON:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ for _, m := range [][2]string{
+ {MDFieldName, metaStripped.Info.Name},
+ {MDFieldVersion, metaStripped.Info.Version},
+ {MDFieldSummary, metaStripped.Info.Summary},
+ {MDFieldDescriptionContentType, metaStripped.Info.DescriptionContentType},
+ {MDFieldKeywords, metaStripped.Info.Keywords},
+ {MDFieldHomePage, metaStripped.Info.HomePage},
+ {MDFieldAuthor, metaStripped.Info.Author},
+ {MDFieldAuthorEmail, metaStripped.Info.AuthorEmail},
+ {MDFieldMaintainer, metaStripped.Info.Maintainer},
+ {MDFieldMaintainerEmail, metaStripped.Info.MaintainerEmail},
+ {MDFieldLicense, metaStripped.Info.License},
+ {MDFieldRequiresPython, metaStripped.Info.RequiresPython},
+ } {
+ recField, jsonField := m[0], m[1]
+ if jsonField == "" {
+ continue
+ }
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: MDFieldToRecField[recField],
+ Value: jsonField,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+
+ for _, m := range []RecFieldToValuesMap{
+ {MDFieldClassifier, metaStripped.Info.Classifier},
+ {MDFieldRequiresDist, metaStripped.Info.RequiresDist},
+ } {
+ for _, v := range m.jsonFields {
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: MDFieldToRecField[m.recField],
+ Value: v,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ }
+ description = metaStripped.Info.Description
+ allReleases = metaStripped.Releases
+ }
+ lines := strings.Split(description, "\n")
+ if len(lines) > 0 {
+ if _, err = wr.WriteFieldMultiline(
+ MDFieldDescription, lines,
+ ); err != nil {
+ log.Fatalln(err)
+ }
+ }
+
+ if !mkdirForPkg(w, r, pkgName) {
+ return false
+ }
+ path := filepath.Join(dirPath, MDFile)
+ existing, err := ioutil.ReadFile(path)
+ if err != nil || bytes.Compare(existing, buf.Bytes()) != 0 {
+ if err = WriteFileSync(dirPath, path, buf.Bytes(), now); err != nil {
+ log.Println("error", r.RemoteAddr, "refresh-json", path, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return false
+ }
+ log.Println(r.RemoteAddr, "pypi", pkgName+"/"+MDFile, "touch")
+ }
+ }
+ mtimes := make(map[string]time.Time)
+ for _, releases := range allReleases {
+ for _, rel := range releases {
+ if rel.Filename == "" || rel.UploadTimeISO8601 == "" {
+ continue
+ }
+ t, err := time.Parse(time.RFC3339Nano, rel.UploadTimeISO8601)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not parse upload_time:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ mtimes[rel.Filename] = t.Truncate(time.Second)
+ }
+ }
+
+ resp, err := c.Do(agentedReq(*PyPIURL + pkgName + "/"))