+ c := http.Client{Transport: &PyPIHTTPTransport}
+ dirPath := filepath.Join(*Root, pkgName)
+ now := time.Now()
+
+ var allReleases map[string][]*PkgReleaseInfo
+ if *JSONURL != "" {
+ resp, err := c.Do(agentedReq(*JSONURL + pkgName + "/json"))
+ if err != nil {
+ log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err)
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return false
+ }
+ if resp.StatusCode != http.StatusOK {
+ resp.Body.Close()
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "HTTP status:", resp.Status,
+ )
+ http.Error(w, "PyPI has non 200 status code", http.StatusBadGateway)
+ return false
+ }
+ body, err := ioutil.ReadAll(resp.Body)
+ resp.Body.Close()
+ var buf bytes.Buffer
+ var description string
+ wr := recfile.NewWriter(&buf)
+ var meta PkgMeta
+ err = json.Unmarshal(body, &meta)
+ if err == nil {
+ for recField, jsonField := range map[string]string{
+ MetadataFieldName: meta.Info.Name,
+ MetadataFieldVersion: meta.Info.Version,
+ MetadataFieldSummary: meta.Info.Summary,
+ MetadataFieldDescriptionContentType: meta.Info.DescriptionContentType,
+ MetadataFieldKeywords: meta.Info.Keywords,
+ MetadataFieldHomePage: meta.Info.HomePage,
+ MetadataFieldAuthor: meta.Info.Author,
+ MetadataFieldAuthorEmail: meta.Info.AuthorEmail,
+ MetadataFieldMaintainer: meta.Info.Maintainer,
+ MetadataFieldMaintainerEmail: meta.Info.MaintainerEmail,
+ MetadataFieldLicense: meta.Info.License,
+ MetadataFieldRequiresPython: meta.Info.RequiresPython,
+ } {
+ if jsonField == "" {
+ continue
+ }
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(recField),
+ Value: jsonField,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ for recField, jsonFields := range map[string][]string{
+ MetadataFieldClassifier: meta.Info.Classifier,
+ MetadataFieldPlatform: meta.Info.Platform,
+ MetadataFieldSupportedPlatform: meta.Info.SupportedPlatform,
+ MetadataFieldRequiresDist: meta.Info.RequiresDist,
+ MetadataFieldRequiresExternal: meta.Info.RequiresExternal,
+ MetadataFieldProjectURL: meta.Info.ProjectURL,
+ MetadataFieldProvidesExtra: meta.Info.ProvidesExtra,
+ } {
+ for _, v := range jsonFields {
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(recField),
+ Value: v,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ }
+ description = meta.Info.Description
+ allReleases = meta.Releases
+ } else {
+ var metaStripped PkgMetaStripped
+ err = json.Unmarshal(body, &metaStripped)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not parse JSON:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ for recField, jsonField := range map[string]string{
+ MetadataFieldName: metaStripped.Info.Name,
+ MetadataFieldVersion: metaStripped.Info.Version,
+ MetadataFieldSummary: metaStripped.Info.Summary,
+ MetadataFieldDescriptionContentType: metaStripped.Info.DescriptionContentType,
+ MetadataFieldKeywords: metaStripped.Info.Keywords,
+ MetadataFieldHomePage: metaStripped.Info.HomePage,
+ MetadataFieldAuthor: metaStripped.Info.Author,
+ MetadataFieldAuthorEmail: metaStripped.Info.AuthorEmail,
+ MetadataFieldMaintainer: metaStripped.Info.Maintainer,
+ MetadataFieldMaintainerEmail: metaStripped.Info.MaintainerEmail,
+ MetadataFieldLicense: metaStripped.Info.License,
+ MetadataFieldRequiresPython: metaStripped.Info.RequiresPython,
+ } {
+ if jsonField == "" {
+ continue
+ }
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(recField),
+ Value: jsonField,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+
+ for recField, jsonFields := range map[string][]string{
+ MetadataFieldClassifier: metaStripped.Info.Classifier,
+ MetadataFieldRequiresDist: metaStripped.Info.RequiresDist,
+ } {
+ for _, v := range jsonFields {
+ if _, err = wr.WriteFields(recfile.Field{
+ Name: metadataFieldToRecField(recField),
+ Value: v,
+ }); err != nil {
+ log.Fatalln(err)
+ }
+ }
+ }
+ description = metaStripped.Info.Description
+ allReleases = metaStripped.Releases
+ }
+ lines := strings.Split(description, "\n")
+ if len(lines) > 0 {
+ if _, err = wr.WriteFieldMultiline(
+ MetadataFieldDescription, lines,
+ ); err != nil {
+ log.Fatalln(err)
+ }
+ }
+
+ if !mkdirForPkg(w, r, pkgName) {
+ return false
+ }
+ path := filepath.Join(dirPath, MetadataFile)
+ existing, err := ioutil.ReadFile(path)
+ if err != nil || bytes.Compare(existing, buf.Bytes()) != 0 {
+ if err = WriteFileSync(dirPath, path, buf.Bytes(), now); err != nil {
+ log.Println("error", r.RemoteAddr, "refresh-json", path, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return false
+ }
+ log.Println(r.RemoteAddr, "pypi", pkgName+"."+MetadataFile, "touch")
+ }
+ }
+ mtimes := make(map[string]time.Time)
+ for _, releases := range allReleases {
+ for _, rel := range releases {
+ if rel.Filename == "" || rel.UploadTimeISO8601 == "" {
+ continue
+ }
+ t, err := time.Parse(time.RFC3339Nano, rel.UploadTimeISO8601)
+ if err != nil {
+ log.Println(
+ "error", r.RemoteAddr, "refresh-json", pkgName,
+ "can not parse upload_time:", err,
+ )
+ http.Error(w, "can not parse metadata JSON", http.StatusBadGateway)
+ return false
+ }
+ mtimes[rel.Filename] = t.Truncate(time.Second)
+ }
+ }
+
+ resp, err := c.Do(agentedReq(*PyPIURL + pkgName + "/"))