list.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2022 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bytes"
  22         _ "embed"
  23         "encoding/hex"
  24         "errors"
  25         "fmt"
  26         "html/template"
  27         "io/fs"
  28         "io/ioutil"
  29         "log"
  30         "net/http"
  31         "os"
  32         "path/filepath"
  33         "sort"
  34         "strconv"
  35         "strings"
  36         "time"
  37 )
  38
  39 // https://warehouse.pypa.io/api-reference/legacy.html
  40 var (
  41         //go:embed root.tmpl
  42         HTMLRootTmplRaw string
  43         HTMLRootTmpl    = template.Must(template.New("root").Parse(HTMLRootTmplRaw))
  44
  45         //go:embed list.tmpl
  46         HTMLReleasesTmplRaw string
  47         HTMLReleasesTmpl    = template.Must(template.New("list").Parse(HTMLReleasesTmplRaw))
  48         KnownExts           = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
  49                 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
  50 )
  51
  52 func listRoot(w http.ResponseWriter, r *http.Request) {
  53         files, err := ioutil.ReadDir(Root)
  54         if err != nil {
  55                 log.Println("error", r.RemoteAddr, "root", err)
  56                 http.Error(w, err.Error(), http.StatusInternalServerError)
  57                 return
  58         }
  59         packages := make([]string, 0, len(files))
  60         for _, f := range files {
  61                 packages = append(packages, f.Name())
  62         }
  63         sort.Strings(packages)
  64         var buf bytes.Buffer
  65         err = HTMLRootTmpl.Execute(&buf, struct {
  66                 RefreshURLPath string
  67                 Packages       []string
  68         }{
  69                 RefreshURLPath: *RefreshURLPath,
  70                 Packages:       packages,
  71         })
  72         if err != nil {
  73                 log.Println("error", r.RemoteAddr, "root", err)
  74                 http.Error(w, err.Error(), http.StatusInternalServerError)
  75                 return
  76         }
  77         w.Write(buf.Bytes())
  78 }
  79
  80 type PkgReleaseInfoByName []*PkgReleaseInfo
  81
  82 func (a PkgReleaseInfoByName) Len() int {
  83         return len(a)
  84 }
  85
  86 func (a PkgReleaseInfoByName) Swap(i, j int) {
  87         a[i], a[j] = a[j], a[i]
  88 }
  89
  90 func (a PkgReleaseInfoByName) Less(i, j int) bool {
  91         if a[i].Version == a[j].Version {
  92                 return a[i].Filename < a[j].Filename
  93         }
  94         return a[i].Version < a[j].Version
  95 }
  96
  97 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
  98 // So here is very simple parser working good enough for most packages
  99 func filenameToVersion(fn string) string {
 100         fn = strings.TrimSuffix(fn, GPGSigExt)
 101         var trimmed string
 102         for _, ext := range KnownExts {
 103                 trimmed = strings.TrimSuffix(fn, ext)
 104                 if trimmed != fn {
 105                         fn = trimmed
 106                         break
 107                 }
 108         }
 109         cols := strings.Split(fn, "-")
 110         for i := 0; i < len(cols); i++ {
 111                 if len(cols[i]) == 0 {
 112                         continue
 113                 }
 114                 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
 115                         return cols[i]
 116                 }
 117         }
 118         if len(cols) > 1 {
 119                 return cols[1]
 120         }
 121         return cols[0]
 122 }
 123
 124 func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) {
 125         dirPath := filepath.Join(Root, pkgName)
 126         entries, err := os.ReadDir(dirPath)
 127         if err != nil {
 128                 return 0, nil, err
 129         }
 130         files := make(map[string]fs.DirEntry, len(entries))
 131         for _, entry := range entries {
 132                 if entry.IsDir() {
 133                         continue
 134                 }
 135                 if entry.Name()[0] == '.' {
 136                         continue
 137                 }
 138                 files[entry.Name()] = entry
 139         }
 140         releaseFiles := make(map[string]*PkgReleaseInfo)
 141         for _, algo := range KnownHashAlgos {
 142                 for fn, entry := range files {
 143                         if Killed {
 144                                 return 0, nil, errors.New("killed")
 145                         }
 146                         if !strings.HasSuffix(fn, "."+algo) {
 147                                 continue
 148                         }
 149                         delete(files, fn)
 150                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 151                         if err != nil {
 152                                 return 0, nil, err
 153                         }
 154                         fnClean := strings.TrimSuffix(fn, "."+algo)
 155                         release := releaseFiles[fnClean]
 156                         if release == nil {
 157                                 fi, err := entry.Info()
 158                                 if err != nil {
 159                                         return 0, nil, err
 160                                 }
 161                                 release = &PkgReleaseInfo{
 162                                         Filename: fnClean,
 163                                         Version:  filenameToVersion(fnClean),
 164                                         UploadTimeISO8601: fi.ModTime().UTC().Truncate(
 165                                                 time.Second,
 166                                         ).Format(time.RFC3339),
 167                                         Digests: make(map[string]string),
 168                                 }
 169                                 releaseFiles[fnClean] = release
 170                                 if entry, exists := files[fnClean]; exists {
 171                                         if doSize {
 172                                                 fi, err := entry.Info()
 173                                                 if err != nil {
 174                                                         return 0, nil, err
 175                                                 }
 176                                                 release.Size = fi.Size()
 177                                         }
 178                                         delete(files, fnClean)
 179                                 }
 180                                 if _, exists := files[fnClean+GPGSigExt]; exists {
 181                                         release.HasSig = true
 182                                         delete(files, fnClean+GPGSigExt)
 183                                 }
 184                         }
 185                         release.Digests[algo] = hex.EncodeToString(digest)
 186                 }
 187         }
 188         releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
 189         for _, release := range releaseFiles {
 190                 releases = append(releases, release)
 191         }
 192         sort.Sort(PkgReleaseInfoByName(releases))
 193         fi, err := os.Stat(dirPath)
 194         if err != nil {
 195                 return 0, nil, err
 196         }
 197         serial := fi.ModTime().Unix()
 198         if fi, err = os.Stat(filepath.Join(dirPath, MDFile)); err == nil {
 199                 serial += fi.ModTime().Unix()
 200         }
 201         return serial, releases, nil
 202 }
 203
 204 func serveListDir(
 205         w http.ResponseWriter,
 206         r *http.Request,
 207         pkgName string,
 208         autorefresh, gpgUpdate bool,
 209 ) {
 210         dirPath := filepath.Join(Root, pkgName)
 211         if autorefresh {
 212                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 213                         return
 214                 }
 215         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
 216                 !refreshDir(w, r, pkgName, "", false) {
 217                 return
 218         }
 219         serial, releases, err := listDir(pkgName, false)
 220         if err != nil {
 221                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 222                 http.Error(w, err.Error(), http.StatusInternalServerError)
 223                 return
 224         }
 225         for _, release := range releases {
 226                 singleDigest := make(map[string]string)
 227                 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
 228                         singleDigest[HashAlgoSHA256] = digest
 229                 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
 230                         singleDigest[HashAlgoSHA512] = digest
 231                 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
 232                         singleDigest[HashAlgoBLAKE2b256] = digest
 233                 } else {
 234                         singleDigest = release.Digests
 235                 }
 236                 release.Digests = singleDigest
 237         }
 238         var buf bytes.Buffer
 239         err = HTMLReleasesTmpl.Execute(&buf, struct {
 240                 RefreshURLPath string
 241                 PkgName        string
 242                 Releases       []*PkgReleaseInfo
 243         }{
 244                 RefreshURLPath: *RefreshURLPath,
 245                 PkgName:        pkgName,
 246                 Releases:       releases,
 247         })
 248         if err != nil {
 249                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 250                 http.Error(w, err.Error(), http.StatusInternalServerError)
 251                 return
 252         }
 253         w.Header().Set("X-PyPI-Last-Serial", strconv.FormatInt(serial, 10))
 254         w.Write(buf.Bytes())
 255         w.Write([]byte(fmt.Sprintf("<!--SERIAL %d-->\n", serial)))
 256 }