list.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2023 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bytes"
  22         _ "embed"
  23         "encoding/hex"
  24         "errors"
  25         "fmt"
  26         "html/template"
  27         "io/fs"
  28         "log"
  29         "net/http"
  30         "os"
  31         "path/filepath"
  32         "sort"
  33         "strconv"
  34         "strings"
  35         "time"
  36 )
  37
  38 // https://warehouse.pypa.io/api-reference/legacy.html
  39 var (
  40         //go:embed root.tmpl
  41         HTMLRootTmplRaw string
  42         HTMLRootTmpl    = template.Must(template.New("root").Parse(HTMLRootTmplRaw))
  43
  44         //go:embed list.tmpl
  45         HTMLReleasesTmplRaw string
  46         HTMLReleasesTmpl    = template.Must(template.New("list").Parse(HTMLReleasesTmplRaw))
  47         KnownExts           = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
  48                 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
  49 )
  50
  51 func listRoot(w http.ResponseWriter, r *http.Request) {
  52         files, err := os.ReadDir(Root)
  53         if err != nil {
  54                 log.Println("error", r.RemoteAddr, "root", err)
  55                 http.Error(w, err.Error(), http.StatusInternalServerError)
  56                 return
  57         }
  58         packages := make([]string, 0, len(files))
  59         for _, f := range files {
  60                 packages = append(packages, f.Name())
  61         }
  62         sort.Strings(packages)
  63         var buf bytes.Buffer
  64         err = HTMLRootTmpl.Execute(&buf, struct {
  65                 RefreshURLPath string
  66                 Packages       []string
  67         }{
  68                 RefreshURLPath: *RefreshURLPath,
  69                 Packages:       packages,
  70         })
  71         if err != nil {
  72                 log.Println("error", r.RemoteAddr, "root", err)
  73                 http.Error(w, err.Error(), http.StatusInternalServerError)
  74                 return
  75         }
  76         w.Write(buf.Bytes())
  77 }
  78
  79 type PkgReleaseInfoByName []*PkgReleaseInfo
  80
  81 func (a PkgReleaseInfoByName) Len() int {
  82         return len(a)
  83 }
  84
  85 func (a PkgReleaseInfoByName) Swap(i, j int) {
  86         a[i], a[j] = a[j], a[i]
  87 }
  88
  89 func (a PkgReleaseInfoByName) Less(i, j int) bool {
  90         if a[i].Version == a[j].Version {
  91                 return a[i].Filename < a[j].Filename
  92         }
  93         return a[i].Version < a[j].Version
  94 }
  95
  96 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
  97 // So here is very simple parser working good enough for most packages
  98 func filenameToVersion(fn string) string {
  99         var trimmed string
 100         for _, ext := range KnownExts {
 101                 trimmed = strings.TrimSuffix(fn, ext)
 102                 if trimmed != fn {
 103                         fn = trimmed
 104                         break
 105                 }
 106         }
 107         cols := strings.Split(fn, "-")
 108         for i := 0; i < len(cols); i++ {
 109                 if len(cols[i]) == 0 {
 110                         continue
 111                 }
 112                 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
 113                         return cols[i]
 114                 }
 115         }
 116         if len(cols) > 1 {
 117                 return cols[1]
 118         }
 119         return cols[0]
 120 }
 121
 122 func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) {
 123         dirPath := filepath.Join(Root, pkgName)
 124         entries, err := os.ReadDir(dirPath)
 125         if err != nil {
 126                 return 0, nil, err
 127         }
 128         files := make(map[string]fs.DirEntry, len(entries))
 129         for _, entry := range entries {
 130                 if entry.IsDir() {
 131                         continue
 132                 }
 133                 if entry.Name()[0] == '.' {
 134                         continue
 135                 }
 136                 files[entry.Name()] = entry
 137         }
 138         releaseFiles := make(map[string]*PkgReleaseInfo)
 139         for _, algo := range KnownHashAlgos {
 140                 for fn, entry := range files {
 141                         if Killed {
 142                                 return 0, nil, errors.New("killed")
 143                         }
 144                         if !strings.HasSuffix(fn, "."+algo) {
 145                                 continue
 146                         }
 147                         delete(files, fn)
 148                         digest, err := os.ReadFile(filepath.Join(dirPath, fn))
 149                         if err != nil {
 150                                 return 0, nil, err
 151                         }
 152                         fnClean := strings.TrimSuffix(fn, "."+algo)
 153                         release := releaseFiles[fnClean]
 154                         if release == nil {
 155                                 fi, err := entry.Info()
 156                                 if err != nil {
 157                                         return 0, nil, err
 158                                 }
 159                                 release = &PkgReleaseInfo{
 160                                         Filename: fnClean,
 161                                         Version:  filenameToVersion(fnClean),
 162                                         UploadTimeISO8601: fi.ModTime().UTC().Truncate(
 163                                                 time.Second,
 164                                         ).Format(time.RFC3339),
 165                                         Digests: make(map[string]string),
 166                                 }
 167                                 releaseFiles[fnClean] = release
 168                                 if entry, exists := files[fnClean]; exists {
 169                                         if doSize {
 170                                                 fi, err := entry.Info()
 171                                                 if err != nil {
 172                                                         return 0, nil, err
 173                                                 }
 174                                                 release.Size = fi.Size()
 175                                         }
 176                                         delete(files, fnClean)
 177                                 }
 178                         }
 179                         release.Digests[algo] = hex.EncodeToString(digest)
 180                 }
 181         }
 182         releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
 183         for _, release := range releaseFiles {
 184                 releases = append(releases, release)
 185         }
 186         sort.Sort(PkgReleaseInfoByName(releases))
 187         fi, err := os.Stat(dirPath)
 188         if err != nil {
 189                 return 0, nil, err
 190         }
 191         serial := fi.ModTime().Unix()
 192         if fi, err = os.Stat(filepath.Join(dirPath, MDFile)); err == nil {
 193                 serial += fi.ModTime().Unix()
 194         }
 195         return serial, releases, nil
 196 }
 197
 198 func serveListDir(
 199         w http.ResponseWriter,
 200         r *http.Request,
 201         pkgName string,
 202         autorefresh bool,
 203 ) {
 204         dirPath := filepath.Join(Root, pkgName)
 205         if autorefresh {
 206                 if !refreshDir(w, r, pkgName, "") {
 207                         return
 208                 }
 209         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
 210                 !refreshDir(w, r, pkgName, "") {
 211                 return
 212         }
 213         serial, releases, err := listDir(pkgName, false)
 214         if err != nil {
 215                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 216                 http.Error(w, err.Error(), http.StatusInternalServerError)
 217                 return
 218         }
 219         for _, release := range releases {
 220                 singleDigest := make(map[string]string)
 221                 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
 222                         singleDigest[HashAlgoSHA256] = digest
 223                 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
 224                         singleDigest[HashAlgoSHA512] = digest
 225                 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
 226                         singleDigest[HashAlgoBLAKE2b256] = digest
 227                 } else {
 228                         singleDigest = release.Digests
 229                 }
 230                 release.Digests = singleDigest
 231         }
 232         var buf bytes.Buffer
 233         err = HTMLReleasesTmpl.Execute(&buf, struct {
 234                 RefreshURLPath string
 235                 PkgName        string
 236                 Releases       []*PkgReleaseInfo
 237         }{
 238                 RefreshURLPath: *RefreshURLPath,
 239                 PkgName:        pkgName,
 240                 Releases:       releases,
 241         })
 242         if err != nil {
 243                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 244                 http.Error(w, err.Error(), http.StatusInternalServerError)
 245                 return
 246         }
 247         w.Header().Set("X-PyPI-Last-Serial", strconv.FormatInt(serial, 10))
 248         w.Write(buf.Bytes())
 249         w.Write([]byte(fmt.Sprintf("<!--SERIAL %d-->\n", serial)))
 250 }