2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
5 This program is free software: you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation, version 3 of the License.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>.
36 // https://warehouse.pypa.io/api-reference/legacy.html
38 HTMLRootTmpl = template.Must(template.New("root").Parse(`<!DOCTYPE html>
41 <meta name="pypi:repository-version" content="1.0">
42 <title>Links for root</title>
44 <body>{{$Refresh := .RefreshURLPath}}{{range .Packages}}
45 <a href="{{$Refresh}}{{.}}/">{{.}}</a><br/>
50 HTMLReleasesTmpl = template.Must(template.New("list").Parse(`<!DOCTYPE html>
53 <meta name="pypi:repository-version" content="1.0">
54 <title>Links for {{.PkgName}}</title>
56 <body>{{$Refresh := .RefreshURLPath}}{{$PkgName := .PkgName}}{{range .Releases}}
57 <a href="{{$Refresh}}{{$PkgName}}/{{.Filename -}}
58 #{{range $a, $d := .Digests}}{{$a}}={{$d}}{{end -}}
59 {{with .HasSig}} data-gpg-sig=true{{end}}">{{.Filename}}</a><br/>
64 KnownExts = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
65 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
68 func listRoot(w http.ResponseWriter, r *http.Request) {
69 files, err := ioutil.ReadDir(*Root)
71 log.Println("error", r.RemoteAddr, "root", err)
72 http.Error(w, err.Error(), http.StatusInternalServerError)
75 packages := make([]string, 0, len(files))
76 for _, f := range files {
77 packages = append(packages, f.Name())
79 sort.Strings(packages)
81 err = HTMLRootTmpl.Execute(&buf, struct {
85 RefreshURLPath: *RefreshURLPath,
89 log.Println("error", r.RemoteAddr, "root", err)
90 http.Error(w, err.Error(), http.StatusInternalServerError)
96 type PkgReleaseInfoByName []*PkgReleaseInfo
98 func (a PkgReleaseInfoByName) Len() int {
102 func (a PkgReleaseInfoByName) Swap(i, j int) {
103 a[i], a[j] = a[j], a[i]
106 func (a PkgReleaseInfoByName) Less(i, j int) bool {
107 if a[i].Version == a[j].Version {
108 return a[i].Filename < a[j].Filename
110 return a[i].Version < a[j].Version
113 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
114 // So here is very simple parser working good enough for most packages
115 func filenameToVersion(fn string) string {
116 fn = strings.TrimSuffix(fn, GPGSigExt)
118 for _, ext := range KnownExts {
119 trimmed = strings.TrimSuffix(fn, ext)
125 cols := strings.Split(fn, "-")
126 for i := 0; i < len(cols); i++ {
127 if len(cols[i]) == 0 {
130 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
140 func listDir(pkgName string, doSize bool) (int, []*PkgReleaseInfo, error) {
141 dirPath := filepath.Join(*Root, pkgName)
142 entries, err := os.ReadDir(dirPath)
146 files := make(map[string]fs.DirEntry, len(entries))
147 for _, entry := range entries {
151 if entry.Name()[0] == '.' {
154 files[entry.Name()] = entry
156 releaseFiles := make(map[string]*PkgReleaseInfo)
157 for _, algo := range KnownHashAlgos {
158 for fn, entry := range files {
160 return 0, nil, errors.New("killed")
162 if !strings.HasSuffix(fn, "."+algo) {
166 digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
170 fnClean := strings.TrimSuffix(fn, "."+algo)
171 release := releaseFiles[fnClean]
173 fi, err := entry.Info()
177 release = &PkgReleaseInfo{
179 Version: filenameToVersion(fnClean),
180 UploadTimeISO8601: fi.ModTime().UTC().Truncate(
182 ).Format(time.RFC3339),
183 Digests: make(map[string]string),
185 releaseFiles[fnClean] = release
186 if entry, exists := files[fnClean]; exists {
188 fi, err := entry.Info()
192 release.Size = fi.Size()
194 delete(files, fnClean)
196 if _, exists := files[fnClean+GPGSigExt]; exists {
197 release.HasSig = true
198 delete(files, fnClean+GPGSigExt)
201 release.Digests[algo] = hex.EncodeToString(digest)
204 releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
205 for _, release := range releaseFiles {
206 releases = append(releases, release)
208 sort.Sort(PkgReleaseInfoByName(releases))
209 return len(entries), releases, nil
213 w http.ResponseWriter,
216 autorefresh, gpgUpdate bool,
218 dirPath := filepath.Join(*Root, pkgName)
220 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
223 } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
224 !refreshDir(w, r, pkgName, "", false) {
227 _, releases, err := listDir(pkgName, false)
229 log.Println("error", r.RemoteAddr, "list", pkgName, err)
230 http.Error(w, err.Error(), http.StatusInternalServerError)
233 for _, release := range releases {
234 singleDigest := make(map[string]string)
235 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
236 singleDigest[HashAlgoSHA256] = digest
237 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
238 singleDigest[HashAlgoSHA512] = digest
239 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
240 singleDigest[HashAlgoBLAKE2b256] = digest
242 singleDigest = release.Digests
244 release.Digests = singleDigest
247 err = HTMLReleasesTmpl.Execute(&buf, struct {
248 RefreshURLPath string
250 Releases []*PkgReleaseInfo
252 RefreshURLPath: *RefreshURLPath,
257 log.Println("error", r.RemoteAddr, "list", pkgName, err)
258 http.Error(w, err.Error(), http.StatusInternalServerError)