X-Git-Url: http://www.git.cypherpunks.ru/?a=blobdiff_plain;f=refresh.go;h=607ef4c50281bea7d5e9fd8bd114a92a1dd056fa;hb=cde8c8c45d508bf29d24365298f3c40c1729514c;hp=59c8ceb9f5dd7c2b6288e27eb1d1cc0c9f6574e9;hpb=3a658c891e7f4f8966a83cd709b2b7468226457f;p=gocheese.git diff --git a/refresh.go b/refresh.go index 59c8ceb..607ef4c 100644 --- a/refresh.go +++ b/refresh.go @@ -1,6 +1,6 @@ /* GoCheese -- Python private package repository and caching proxy -Copyright (C) 2019-2021 Sergey Matveev +Copyright (C) 2019-2022 Sergey Matveev This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -25,6 +25,7 @@ import ( "crypto/sha512" "encoding/hex" "encoding/json" + "errors" "hash" "io" "io/ioutil" @@ -115,6 +116,12 @@ func refreshDir( return false } body, err := ioutil.ReadAll(resp.Body) + if err != nil { + resp.Body.Close() + log.Println("error", r.RemoteAddr, "refresh-json", pkgName, err) + http.Error(w, "can not read body", http.StatusBadGateway) + return false + } resp.Body.Close() var buf bytes.Buffer var description string @@ -123,42 +130,42 @@ func refreshDir( err = json.Unmarshal(body, &meta) if err == nil { for _, m := range [][2]string{ - {MetadataFieldName, meta.Info.Name}, - {MetadataFieldVersion, meta.Info.Version}, - {MetadataFieldSummary, meta.Info.Summary}, - {MetadataFieldDescriptionContentType, meta.Info.DescriptionContentType}, - {MetadataFieldKeywords, meta.Info.Keywords}, - {MetadataFieldHomePage, meta.Info.HomePage}, - {MetadataFieldAuthor, meta.Info.Author}, - {MetadataFieldAuthorEmail, meta.Info.AuthorEmail}, - {MetadataFieldMaintainer, meta.Info.Maintainer}, - {MetadataFieldMaintainerEmail, meta.Info.MaintainerEmail}, - {MetadataFieldLicense, meta.Info.License}, - {MetadataFieldRequiresPython, meta.Info.RequiresPython}, + {MDFieldName, meta.Info.Name}, + {MDFieldVersion, meta.Info.Version}, + {MDFieldSummary, meta.Info.Summary}, + {MDFieldDescriptionContentType, meta.Info.DescriptionContentType}, + {MDFieldKeywords, meta.Info.Keywords}, + {MDFieldHomePage, meta.Info.HomePage}, + {MDFieldAuthor, meta.Info.Author}, + {MDFieldAuthorEmail, meta.Info.AuthorEmail}, + {MDFieldMaintainer, meta.Info.Maintainer}, + {MDFieldMaintainerEmail, meta.Info.MaintainerEmail}, + {MDFieldLicense, meta.Info.License}, + {MDFieldRequiresPython, meta.Info.RequiresPython}, } { recField, jsonField := m[0], m[1] if jsonField == "" { continue } if _, err = wr.WriteFields(recfile.Field{ - Name: metadataFieldToRecField(recField), + Name: MDFieldToRecField[recField], Value: jsonField, }); err != nil { log.Fatalln(err) } } for _, m := range []RecFieldToValuesMap{ - {MetadataFieldClassifier, meta.Info.Classifier}, - {MetadataFieldPlatform, meta.Info.Platform}, - {MetadataFieldSupportedPlatform, meta.Info.SupportedPlatform}, - {MetadataFieldRequiresDist, meta.Info.RequiresDist}, - {MetadataFieldRequiresExternal, meta.Info.RequiresExternal}, - {MetadataFieldProjectURL, meta.Info.ProjectURL}, - {MetadataFieldProvidesExtra, meta.Info.ProvidesExtra}, + {MDFieldClassifier, meta.Info.Classifier}, + {MDFieldPlatform, meta.Info.Platform}, + {MDFieldSupportedPlatform, meta.Info.SupportedPlatform}, + {MDFieldRequiresDist, meta.Info.RequiresDist}, + {MDFieldRequiresExternal, meta.Info.RequiresExternal}, + {MDFieldProjectURL, meta.Info.ProjectURL}, + {MDFieldProvidesExtra, meta.Info.ProvidesExtra}, } { for _, v := range m.jsonFields { if _, err = wr.WriteFields(recfile.Field{ - Name: metadataFieldToRecField(m.recField), + Name: MDFieldToRecField[m.recField], Value: v, }); err != nil { log.Fatalln(err) @@ -179,25 +186,25 @@ func refreshDir( return false } for _, m := range [][2]string{ - {MetadataFieldName, metaStripped.Info.Name}, - {MetadataFieldVersion, metaStripped.Info.Version}, - {MetadataFieldSummary, metaStripped.Info.Summary}, - {MetadataFieldDescriptionContentType, metaStripped.Info.DescriptionContentType}, - {MetadataFieldKeywords, metaStripped.Info.Keywords}, - {MetadataFieldHomePage, metaStripped.Info.HomePage}, - {MetadataFieldAuthor, metaStripped.Info.Author}, - {MetadataFieldAuthorEmail, metaStripped.Info.AuthorEmail}, - {MetadataFieldMaintainer, metaStripped.Info.Maintainer}, - {MetadataFieldMaintainerEmail, metaStripped.Info.MaintainerEmail}, - {MetadataFieldLicense, metaStripped.Info.License}, - {MetadataFieldRequiresPython, metaStripped.Info.RequiresPython}, + {MDFieldName, metaStripped.Info.Name}, + {MDFieldVersion, metaStripped.Info.Version}, + {MDFieldSummary, metaStripped.Info.Summary}, + {MDFieldDescriptionContentType, metaStripped.Info.DescriptionContentType}, + {MDFieldKeywords, metaStripped.Info.Keywords}, + {MDFieldHomePage, metaStripped.Info.HomePage}, + {MDFieldAuthor, metaStripped.Info.Author}, + {MDFieldAuthorEmail, metaStripped.Info.AuthorEmail}, + {MDFieldMaintainer, metaStripped.Info.Maintainer}, + {MDFieldMaintainerEmail, metaStripped.Info.MaintainerEmail}, + {MDFieldLicense, metaStripped.Info.License}, + {MDFieldRequiresPython, metaStripped.Info.RequiresPython}, } { recField, jsonField := m[0], m[1] if jsonField == "" { continue } if _, err = wr.WriteFields(recfile.Field{ - Name: metadataFieldToRecField(recField), + Name: MDFieldToRecField[recField], Value: jsonField, }); err != nil { log.Fatalln(err) @@ -205,12 +212,12 @@ func refreshDir( } for _, m := range []RecFieldToValuesMap{ - {MetadataFieldClassifier, metaStripped.Info.Classifier}, - {MetadataFieldRequiresDist, metaStripped.Info.RequiresDist}, + {MDFieldClassifier, metaStripped.Info.Classifier}, + {MDFieldRequiresDist, metaStripped.Info.RequiresDist}, } { for _, v := range m.jsonFields { if _, err = wr.WriteFields(recfile.Field{ - Name: metadataFieldToRecField(m.recField), + Name: MDFieldToRecField[m.recField], Value: v, }); err != nil { log.Fatalln(err) @@ -223,7 +230,7 @@ func refreshDir( lines := strings.Split(description, "\n") if len(lines) > 0 { if _, err = wr.WriteFieldMultiline( - MetadataFieldDescription, lines, + MDFieldDescription, lines, ); err != nil { log.Fatalln(err) } @@ -232,7 +239,7 @@ func refreshDir( if !mkdirForPkg(w, r, pkgName) { return false } - path := filepath.Join(dirPath, MetadataFile) + path := filepath.Join(dirPath, MDFile) existing, err := ioutil.ReadFile(path) if err != nil || bytes.Compare(existing, buf.Bytes()) != 0 { if err = WriteFileSync(dirPath, path, buf.Bytes(), now); err != nil { @@ -240,7 +247,7 @@ func refreshDir( http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "pypi", pkgName+"/"+MetadataFile, "touch") + log.Println(r.RemoteAddr, "pypi", pkgName+"/"+MDFile, "touch") } } mtimes := make(map[string]time.Time) @@ -427,6 +434,15 @@ func refreshDir( http.Error(w, "digest mismatch", http.StatusBadGateway) return false } + if digestStored, err := ioutil.ReadFile(path + "." + hashAlgo); err == nil && + bytes.Compare(digest, digestStored) != 0 { + err = errors.New("stored digest mismatch") + log.Println("error", r.RemoteAddr, "pypi", filename, err) + os.Remove(dst.Name()) + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return false + } if !NoSync { if err = dst.Sync(); err != nil { os.Remove(dst.Name()) @@ -507,9 +523,6 @@ func refreshDir( } if filename == filenameGet || gpgUpdate { - if _, err = os.Stat(path); err != nil { - goto GPGSigSkip - } resp, err := c.Do(agentedReq(uri + GPGSigExt)) if err != nil { goto GPGSigSkip