list.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2022 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bytes"
  22         _ "embed"
  23         "encoding/hex"
  24         "errors"
  25         "fmt"
  26         "html/template"
  27         "io/fs"
  28         "log"
  29         "net/http"
  30         "os"
  31         "path/filepath"
  32         "sort"
  33         "strconv"
  34         "strings"
  35         "time"
  36 )
  37
  38 // https://warehouse.pypa.io/api-reference/legacy.html
  39 var (
  40         //go:embed root.tmpl
  41         HTMLRootTmplRaw string
  42         HTMLRootTmpl    = template.Must(template.New("root").Parse(HTMLRootTmplRaw))
  43
  44         //go:embed list.tmpl
  45         HTMLReleasesTmplRaw string
  46         HTMLReleasesTmpl    = template.Must(template.New("list").Parse(HTMLReleasesTmplRaw))
  47         KnownExts           = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
  48                 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
  49 )
  50
  51 func listRoot(w http.ResponseWriter, r *http.Request) {
  52         files, err := os.ReadDir(Root)
  53         if err != nil {
  54                 log.Println("error", r.RemoteAddr, "root", err)
  55                 http.Error(w, err.Error(), http.StatusInternalServerError)
  56                 return
  57         }
  58         packages := make([]string, 0, len(files))
  59         for _, f := range files {
  60                 packages = append(packages, f.Name())
  61         }
  62         sort.Strings(packages)
  63         var buf bytes.Buffer
  64         err = HTMLRootTmpl.Execute(&buf, struct {
  65                 RefreshURLPath string
  66                 Packages       []string
  67         }{
  68                 RefreshURLPath: *RefreshURLPath,
  69                 Packages:       packages,
  70         })
  71         if err != nil {
  72                 log.Println("error", r.RemoteAddr, "root", err)
  73                 http.Error(w, err.Error(), http.StatusInternalServerError)
  74                 return
  75         }
  76         w.Write(buf.Bytes())
  77 }
  78
  79 type PkgReleaseInfoByName []*PkgReleaseInfo
  80
  81 func (a PkgReleaseInfoByName) Len() int {
  82         return len(a)
  83 }
  84
  85 func (a PkgReleaseInfoByName) Swap(i, j int) {
  86         a[i], a[j] = a[j], a[i]
  87 }
  88
  89 func (a PkgReleaseInfoByName) Less(i, j int) bool {
  90         if a[i].Version == a[j].Version {
  91                 return a[i].Filename < a[j].Filename
  92         }
  93         return a[i].Version < a[j].Version
  94 }
  95
  96 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
  97 // So here is very simple parser working good enough for most packages
  98 func filenameToVersion(fn string) string {
  99         fn = strings.TrimSuffix(fn, GPGSigExt)
 100         var trimmed string
 101         for _, ext := range KnownExts {
 102                 trimmed = strings.TrimSuffix(fn, ext)
 103                 if trimmed != fn {
 104                         fn = trimmed
 105                         break
 106                 }
 107         }
 108         cols := strings.Split(fn, "-")
 109         for i := 0; i < len(cols); i++ {
 110                 if len(cols[i]) == 0 {
 111                         continue
 112                 }
 113                 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
 114                         return cols[i]
 115                 }
 116         }
 117         if len(cols) > 1 {
 118                 return cols[1]
 119         }
 120         return cols[0]
 121 }
 122
 123 func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) {
 124         dirPath := filepath.Join(Root, pkgName)
 125         entries, err := os.ReadDir(dirPath)
 126         if err != nil {
 127                 return 0, nil, err
 128         }
 129         files := make(map[string]fs.DirEntry, len(entries))
 130         for _, entry := range entries {
 131                 if entry.IsDir() {
 132                         continue
 133                 }
 134                 if entry.Name()[0] == '.' {
 135                         continue
 136                 }
 137                 files[entry.Name()] = entry
 138         }
 139         releaseFiles := make(map[string]*PkgReleaseInfo)
 140         for _, algo := range KnownHashAlgos {
 141                 for fn, entry := range files {
 142                         if Killed {
 143                                 return 0, nil, errors.New("killed")
 144                         }
 145                         if !strings.HasSuffix(fn, "."+algo) {
 146                                 continue
 147                         }
 148                         delete(files, fn)
 149                         digest, err := os.ReadFile(filepath.Join(dirPath, fn))
 150                         if err != nil {
 151                                 return 0, nil, err
 152                         }
 153                         fnClean := strings.TrimSuffix(fn, "."+algo)
 154                         release := releaseFiles[fnClean]
 155                         if release == nil {
 156                                 fi, err := entry.Info()
 157                                 if err != nil {
 158                                         return 0, nil, err
 159                                 }
 160                                 release = &PkgReleaseInfo{
 161                                         Filename: fnClean,
 162                                         Version:  filenameToVersion(fnClean),
 163                                         UploadTimeISO8601: fi.ModTime().UTC().Truncate(
 164                                                 time.Second,
 165                                         ).Format(time.RFC3339),
 166                                         Digests: make(map[string]string),
 167                                 }
 168                                 releaseFiles[fnClean] = release
 169                                 if entry, exists := files[fnClean]; exists {
 170                                         if doSize {
 171                                                 fi, err := entry.Info()
 172                                                 if err != nil {
 173                                                         return 0, nil, err
 174                                                 }
 175                                                 release.Size = fi.Size()
 176                                         }
 177                                         delete(files, fnClean)
 178                                 }
 179                                 if _, exists := files[fnClean+GPGSigExt]; exists {
 180                                         release.HasSig = true
 181                                         delete(files, fnClean+GPGSigExt)
 182                                 }
 183                         }
 184                         release.Digests[algo] = hex.EncodeToString(digest)
 185                 }
 186         }
 187         releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
 188         for _, release := range releaseFiles {
 189                 releases = append(releases, release)
 190         }
 191         sort.Sort(PkgReleaseInfoByName(releases))
 192         fi, err := os.Stat(dirPath)
 193         if err != nil {
 194                 return 0, nil, err
 195         }
 196         serial := fi.ModTime().Unix()
 197         if fi, err = os.Stat(filepath.Join(dirPath, MDFile)); err == nil {
 198                 serial += fi.ModTime().Unix()
 199         }
 200         return serial, releases, nil
 201 }
 202
 203 func serveListDir(
 204         w http.ResponseWriter,
 205         r *http.Request,
 206         pkgName string,
 207         autorefresh, gpgUpdate bool,
 208 ) {
 209         dirPath := filepath.Join(Root, pkgName)
 210         if autorefresh {
 211                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 212                         return
 213                 }
 214         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
 215                 !refreshDir(w, r, pkgName, "", false) {
 216                 return
 217         }
 218         serial, releases, err := listDir(pkgName, false)
 219         if err != nil {
 220                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 221                 http.Error(w, err.Error(), http.StatusInternalServerError)
 222                 return
 223         }
 224         for _, release := range releases {
 225                 singleDigest := make(map[string]string)
 226                 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
 227                         singleDigest[HashAlgoSHA256] = digest
 228                 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
 229                         singleDigest[HashAlgoSHA512] = digest
 230                 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
 231                         singleDigest[HashAlgoBLAKE2b256] = digest
 232                 } else {
 233                         singleDigest = release.Digests
 234                 }
 235                 release.Digests = singleDigest
 236         }
 237         var buf bytes.Buffer
 238         err = HTMLReleasesTmpl.Execute(&buf, struct {
 239                 RefreshURLPath string
 240                 PkgName        string
 241                 Releases       []*PkgReleaseInfo
 242         }{
 243                 RefreshURLPath: *RefreshURLPath,
 244                 PkgName:        pkgName,
 245                 Releases:       releases,
 246         })
 247         if err != nil {
 248                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 249                 http.Error(w, err.Error(), http.StatusInternalServerError)
 250                 return
 251         }
 252         w.Header().Set("X-PyPI-Last-Serial", strconv.FormatInt(serial, 10))
 253         w.Write(buf.Bytes())
 254         w.Write([]byte(fmt.Sprintf("<!--SERIAL %d-->\n", serial)))
 255 }