list.go

   1 // GoCheese -- Python private package repository and caching proxy
   2 // Copyright (C) 2019-2024 Sergey Matveev <stargrave@stargrave.org>
   3 //
   4 // This program is free software: you can redistribute it and/or modify
   5 // it under the terms of the GNU General Public License as published by
   6 // the Free Software Foundation, version 3 of the License.
   7 //
   8 // This program is distributed in the hope that it will be useful,
   9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 // GNU General Public License for more details.
  12 //
  13 // You should have received a copy of the GNU General Public License
  14 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15
  16 package main
  17
  18 import (
  19         "bytes"
  20         _ "embed"
  21         "encoding/hex"
  22         "errors"
  23         "fmt"
  24         "html/template"
  25         "io/fs"
  26         "log"
  27         "net/http"
  28         "os"
  29         "path/filepath"
  30         "sort"
  31         "strconv"
  32         "strings"
  33         "time"
  34 )
  35
  36 // https://warehouse.pypa.io/api-reference/legacy.html
  37 var (
  38         //go:embed root.tmpl
  39         HTMLRootTmplRaw string
  40         HTMLRootTmpl    = template.Must(template.New("root").Parse(HTMLRootTmplRaw))
  41
  42         //go:embed list.tmpl
  43         HTMLReleasesTmplRaw string
  44         HTMLReleasesTmpl    = template.Must(template.New("list").Parse(HTMLReleasesTmplRaw))
  45         KnownExts           = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
  46                 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
  47 )
  48
  49 func listRoot(w http.ResponseWriter, r *http.Request) {
  50         files, err := os.ReadDir(Root)
  51         if err != nil {
  52                 log.Println("error", r.RemoteAddr, "root", err)
  53                 http.Error(w, err.Error(), http.StatusInternalServerError)
  54                 return
  55         }
  56         packages := make([]string, 0, len(files))
  57         for _, f := range files {
  58                 packages = append(packages, f.Name())
  59         }
  60         sort.Strings(packages)
  61         var buf bytes.Buffer
  62         err = HTMLRootTmpl.Execute(&buf, struct {
  63                 RefreshURLPath string
  64                 Packages       []string
  65         }{
  66                 RefreshURLPath: *RefreshURLPath,
  67                 Packages:       packages,
  68         })
  69         if err != nil {
  70                 log.Println("error", r.RemoteAddr, "root", err)
  71                 http.Error(w, err.Error(), http.StatusInternalServerError)
  72                 return
  73         }
  74         w.Write(buf.Bytes())
  75 }
  76
  77 type PkgReleaseInfoByName []*PkgReleaseInfo
  78
  79 func (a PkgReleaseInfoByName) Len() int {
  80         return len(a)
  81 }
  82
  83 func (a PkgReleaseInfoByName) Swap(i, j int) {
  84         a[i], a[j] = a[j], a[i]
  85 }
  86
  87 func (a PkgReleaseInfoByName) Less(i, j int) bool {
  88         if a[i].Version == a[j].Version {
  89                 return a[i].Filename < a[j].Filename
  90         }
  91         return a[i].Version < a[j].Version
  92 }
  93
  94 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
  95 // So here is very simple parser working good enough for most packages
  96 func filenameToVersion(fn string) string {
  97         var trimmed string
  98         for _, ext := range KnownExts {
  99                 trimmed = strings.TrimSuffix(fn, ext)
 100                 if trimmed != fn {
 101                         fn = trimmed
 102                         break
 103                 }
 104         }
 105         cols := strings.Split(fn, "-")
 106         for i := 0; i < len(cols); i++ {
 107                 if len(cols[i]) == 0 {
 108                         continue
 109                 }
 110                 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
 111                         return cols[i]
 112                 }
 113         }
 114         if len(cols) > 1 {
 115                 return cols[1]
 116         }
 117         return cols[0]
 118 }
 119
 120 func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) {
 121         dirPath := filepath.Join(Root, pkgName)
 122         entries, err := os.ReadDir(dirPath)
 123         if err != nil {
 124                 return 0, nil, err
 125         }
 126         files := make(map[string]fs.DirEntry, len(entries))
 127         for _, entry := range entries {
 128                 if entry.IsDir() {
 129                         continue
 130                 }
 131                 if entry.Name()[0] == '.' {
 132                         continue
 133                 }
 134                 files[entry.Name()] = entry
 135         }
 136         releaseFiles := make(map[string]*PkgReleaseInfo)
 137         for _, algo := range KnownHashAlgos {
 138                 for fn, entry := range files {
 139                         if Killed {
 140                                 return 0, nil, errors.New("killed")
 141                         }
 142                         if !strings.HasSuffix(fn, "."+algo) {
 143                                 continue
 144                         }
 145                         delete(files, fn)
 146                         digest, err := os.ReadFile(filepath.Join(dirPath, fn))
 147                         if err != nil {
 148                                 return 0, nil, err
 149                         }
 150                         fnClean := strings.TrimSuffix(fn, "."+algo)
 151                         release := releaseFiles[fnClean]
 152                         if release == nil {
 153                                 fi, err := entry.Info()
 154                                 if err != nil {
 155                                         return 0, nil, err
 156                                 }
 157                                 release = &PkgReleaseInfo{
 158                                         Filename: fnClean,
 159                                         Version:  filenameToVersion(fnClean),
 160                                         UploadTimeISO8601: fi.ModTime().UTC().Truncate(
 161                                                 time.Second,
 162                                         ).Format(time.RFC3339),
 163                                         Digests: make(map[string]string),
 164                                 }
 165                                 releaseFiles[fnClean] = release
 166                                 if entry, exists := files[fnClean]; exists {
 167                                         if doSize {
 168                                                 fi, err := entry.Info()
 169                                                 if err != nil {
 170                                                         return 0, nil, err
 171                                                 }
 172                                                 release.Size = fi.Size()
 173                                         }
 174                                         delete(files, fnClean)
 175                                 }
 176                         }
 177                         release.Digests[algo] = hex.EncodeToString(digest)
 178                 }
 179         }
 180         releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
 181         for _, release := range releaseFiles {
 182                 releases = append(releases, release)
 183         }
 184         sort.Sort(PkgReleaseInfoByName(releases))
 185         fi, err := os.Stat(dirPath)
 186         if err != nil {
 187                 return 0, nil, err
 188         }
 189         serial := fi.ModTime().Unix()
 190         if fi, err = os.Stat(filepath.Join(dirPath, MDFile)); err == nil {
 191                 serial += fi.ModTime().Unix()
 192         }
 193         return serial, releases, nil
 194 }
 195
 196 func serveListDir(
 197         w http.ResponseWriter,
 198         r *http.Request,
 199         pkgName string,
 200         autorefresh bool,
 201 ) {
 202         dirPath := filepath.Join(Root, pkgName)
 203         if autorefresh {
 204                 if !refreshDir(w, r, pkgName, "") {
 205                         return
 206                 }
 207         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
 208                 !refreshDir(w, r, pkgName, "") {
 209                 return
 210         }
 211         serial, releases, err := listDir(pkgName, false)
 212         if err != nil {
 213                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 214                 http.Error(w, err.Error(), http.StatusInternalServerError)
 215                 return
 216         }
 217         for _, release := range releases {
 218                 singleDigest := make(map[string]string)
 219                 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
 220                         singleDigest[HashAlgoSHA256] = digest
 221                 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
 222                         singleDigest[HashAlgoSHA512] = digest
 223                 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
 224                         singleDigest[HashAlgoBLAKE2b256] = digest
 225                 } else {
 226                         singleDigest = release.Digests
 227                 }
 228                 release.Digests = singleDigest
 229         }
 230         var buf bytes.Buffer
 231         err = HTMLReleasesTmpl.Execute(&buf, struct {
 232                 RefreshURLPath string
 233                 PkgName        string
 234                 Releases       []*PkgReleaseInfo
 235         }{
 236                 RefreshURLPath: *RefreshURLPath,
 237                 PkgName:        pkgName,
 238                 Releases:       releases,
 239         })
 240         if err != nil {
 241                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 242                 http.Error(w, err.Error(), http.StatusInternalServerError)
 243                 return
 244         }
 245         w.Header().Set("X-PyPI-Last-Serial", strconv.FormatInt(serial, 10))
 246         w.Write(buf.Bytes())
 247         w.Write([]byte(fmt.Sprintf("<!--SERIAL %d-->\n", serial)))
 248 }