// GoCheese -- Python private package repository and caching proxy // Copyright (C) 2019-2024 Sergey Matveev // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, version 3 of the License. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . package main import ( "bytes" _ "embed" "encoding/hex" "errors" "fmt" "html/template" "io/fs" "log" "net/http" "os" "path/filepath" "sort" "strconv" "strings" "time" ) // https://warehouse.pypa.io/api-reference/legacy.html var ( //go:embed root.tmpl HTMLRootTmplRaw string HTMLRootTmpl = template.Must(template.New("root").Parse(HTMLRootTmplRaw)) //go:embed list.tmpl HTMLReleasesTmplRaw string HTMLReleasesTmpl = template.Must(template.New("list").Parse(HTMLReleasesTmplRaw)) KnownExts = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg", ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"} ) func listRoot(w http.ResponseWriter, r *http.Request) { files, err := os.ReadDir(Root) if err != nil { log.Println("error", r.RemoteAddr, "root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } packages := make([]string, 0, len(files)) for _, f := range files { packages = append(packages, f.Name()) } sort.Strings(packages) var buf bytes.Buffer err = HTMLRootTmpl.Execute(&buf, struct { RefreshURLPath string Packages []string }{ RefreshURLPath: *RefreshURLPath, Packages: packages, }) if err != nil { log.Println("error", r.RemoteAddr, "root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Write(buf.Bytes()) } type PkgReleaseInfoByName []*PkgReleaseInfo func (a PkgReleaseInfoByName) Len() int { return len(a) } func (a PkgReleaseInfoByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a PkgReleaseInfoByName) Less(i, j int) bool { if a[i].Version == a[j].Version { return a[i].Filename < a[j].Filename } return a[i].Version < a[j].Version } // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/ // So here is very simple parser working good enough for most packages func filenameToVersion(fn string) string { var trimmed string for _, ext := range KnownExts { trimmed = strings.TrimSuffix(fn, ext) if trimmed != fn { fn = trimmed break } } cols := strings.Split(fn, "-") for i := 0; i < len(cols); i++ { if len(cols[i]) == 0 { continue } if ('0' <= cols[i][0]) && (cols[i][0] <= '9') { return cols[i] } } if len(cols) > 1 { return cols[1] } return cols[0] } func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) { dirPath := filepath.Join(Root, pkgName) entries, err := os.ReadDir(dirPath) if err != nil { return 0, nil, err } files := make(map[string]fs.DirEntry, len(entries)) for _, entry := range entries { if entry.IsDir() { continue } if entry.Name()[0] == '.' { continue } files[entry.Name()] = entry } releaseFiles := make(map[string]*PkgReleaseInfo) for _, algo := range KnownHashAlgos { for fn, entry := range files { if Killed { return 0, nil, errors.New("killed") } if !strings.HasSuffix(fn, "."+algo) { continue } delete(files, fn) digest, err := os.ReadFile(filepath.Join(dirPath, fn)) if err != nil { return 0, nil, err } fnClean := strings.TrimSuffix(fn, "."+algo) release := releaseFiles[fnClean] if release == nil { fi, err := entry.Info() if err != nil { return 0, nil, err } release = &PkgReleaseInfo{ Filename: fnClean, Version: filenameToVersion(fnClean), UploadTimeISO8601: fi.ModTime().UTC().Truncate( time.Second, ).Format(time.RFC3339), Digests: make(map[string]string), } releaseFiles[fnClean] = release if entry, exists := files[fnClean]; exists { if doSize { fi, err := entry.Info() if err != nil { return 0, nil, err } release.Size = fi.Size() } delete(files, fnClean) } } release.Digests[algo] = hex.EncodeToString(digest) } } releases := make([]*PkgReleaseInfo, 0, len(releaseFiles)) for _, release := range releaseFiles { releases = append(releases, release) } sort.Sort(PkgReleaseInfoByName(releases)) fi, err := os.Stat(dirPath) if err != nil { return 0, nil, err } serial := fi.ModTime().Unix() if fi, err = os.Stat(filepath.Join(dirPath, MDFile)); err == nil { serial += fi.ModTime().Unix() } return serial, releases, nil } func serveListDir( w http.ResponseWriter, r *http.Request, pkgName string, autorefresh bool, ) { dirPath := filepath.Join(Root, pkgName) if autorefresh { if !refreshDir(w, r, pkgName, "") { return } } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "") { return } serial, releases, err := listDir(pkgName, false) if err != nil { log.Println("error", r.RemoteAddr, "list", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } for _, release := range releases { singleDigest := make(map[string]string) if digest, exists := release.Digests[HashAlgoSHA256]; exists { singleDigest[HashAlgoSHA256] = digest } else if digest, exists := release.Digests[HashAlgoSHA512]; exists { singleDigest[HashAlgoSHA512] = digest } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists { singleDigest[HashAlgoBLAKE2b256] = digest } else { singleDigest = release.Digests } release.Digests = singleDigest } var buf bytes.Buffer err = HTMLReleasesTmpl.Execute(&buf, struct { RefreshURLPath string PkgName string Releases []*PkgReleaseInfo }{ RefreshURLPath: *RefreshURLPath, PkgName: pkgName, Releases: releases, }) if err != nil { log.Println("error", r.RemoteAddr, "list", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } w.Header().Set("X-PyPI-Last-Serial", strconv.FormatInt(serial, 10)) w.Write(buf.Bytes()) w.Write([]byte(fmt.Sprintf("\n", serial))) }