list.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bytes"
  22         "encoding/hex"
  23         "errors"
  24         "fmt"
  25         "html/template"
  26         "io/fs"
  27         "io/ioutil"
  28         "log"
  29         "net/http"
  30         "os"
  31         "path/filepath"
  32         "sort"
  33         "strconv"
  34         "strings"
  35         "time"
  36 )
  37
  38 // https://warehouse.pypa.io/api-reference/legacy.html
  39 var (
  40         HTMLRootTmpl = template.Must(template.New("root").Parse(`<!DOCTYPE html>
  41 <html>
  42   <head>
  43     <meta name="pypi:repository-version" content="1.0">
  44     <title>Simple index</title>
  45   </head>
  46   <body>{{$Refresh := .RefreshURLPath}}{{range .Packages}}
  47     <a href="{{$Refresh}}{{.}}/">{{.}}</a><br/>
  48 {{- end}}
  49   </body>
  50 </html>
  51 `))
  52         HTMLReleasesTmpl = template.Must(template.New("list").Parse(`<!DOCTYPE html>
  53 <html>
  54   <head>
  55     <meta name="pypi:repository-version" content="1.0">
  56     <title>Links for {{.PkgName}}</title>
  57   </head>
  58   <body>{{$Refresh := .RefreshURLPath}}{{$PkgName := .PkgName}}{{range .Releases}}
  59     <a href="{{$Refresh}}{{$PkgName}}/{{.Filename -}}
  60         #{{range $a, $d := .Digests}}{{$a}}={{$d}}{{end -}}
  61         {{with .HasSig}} data-gpg-sig=true{{end}}">{{.Filename}}</a><br/>
  62 {{- end}}
  63   </body>
  64 </html>
  65 `))
  66         KnownExts = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
  67                 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
  68 )
  69
  70 func listRoot(w http.ResponseWriter, r *http.Request) {
  71         files, err := ioutil.ReadDir(Root)
  72         if err != nil {
  73                 log.Println("error", r.RemoteAddr, "root", err)
  74                 http.Error(w, err.Error(), http.StatusInternalServerError)
  75                 return
  76         }
  77         packages := make([]string, 0, len(files))
  78         for _, f := range files {
  79                 packages = append(packages, f.Name())
  80         }
  81         sort.Strings(packages)
  82         var buf bytes.Buffer
  83         err = HTMLRootTmpl.Execute(&buf, struct {
  84                 RefreshURLPath string
  85                 Packages       []string
  86         }{
  87                 RefreshURLPath: *RefreshURLPath,
  88                 Packages:       packages,
  89         })
  90         if err != nil {
  91                 log.Println("error", r.RemoteAddr, "root", err)
  92                 http.Error(w, err.Error(), http.StatusInternalServerError)
  93                 return
  94         }
  95         w.Write(buf.Bytes())
  96 }
  97
  98 type PkgReleaseInfoByName []*PkgReleaseInfo
  99
 100 func (a PkgReleaseInfoByName) Len() int {
 101         return len(a)
 102 }
 103
 104 func (a PkgReleaseInfoByName) Swap(i, j int) {
 105         a[i], a[j] = a[j], a[i]
 106 }
 107
 108 func (a PkgReleaseInfoByName) Less(i, j int) bool {
 109         if a[i].Version == a[j].Version {
 110                 return a[i].Filename < a[j].Filename
 111         }
 112         return a[i].Version < a[j].Version
 113 }
 114
 115 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
 116 // So here is very simple parser working good enough for most packages
 117 func filenameToVersion(fn string) string {
 118         fn = strings.TrimSuffix(fn, GPGSigExt)
 119         var trimmed string
 120         for _, ext := range KnownExts {
 121                 trimmed = strings.TrimSuffix(fn, ext)
 122                 if trimmed != fn {
 123                         fn = trimmed
 124                         break
 125                 }
 126         }
 127         cols := strings.Split(fn, "-")
 128         for i := 0; i < len(cols); i++ {
 129                 if len(cols[i]) == 0 {
 130                         continue
 131                 }
 132                 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
 133                         return cols[i]
 134                 }
 135         }
 136         if len(cols) > 1 {
 137                 return cols[1]
 138         }
 139         return cols[0]
 140 }
 141
 142 func listDir(pkgName string, doSize bool) (int64, []*PkgReleaseInfo, error) {
 143         dirPath := filepath.Join(Root, pkgName)
 144         entries, err := os.ReadDir(dirPath)
 145         if err != nil {
 146                 return 0, nil, err
 147         }
 148         files := make(map[string]fs.DirEntry, len(entries))
 149         for _, entry := range entries {
 150                 if entry.IsDir() {
 151                         continue
 152                 }
 153                 if entry.Name()[0] == '.' {
 154                         continue
 155                 }
 156                 files[entry.Name()] = entry
 157         }
 158         releaseFiles := make(map[string]*PkgReleaseInfo)
 159         for _, algo := range KnownHashAlgos {
 160                 for fn, entry := range files {
 161                         if Killed {
 162                                 return 0, nil, errors.New("killed")
 163                         }
 164                         if !strings.HasSuffix(fn, "."+algo) {
 165                                 continue
 166                         }
 167                         delete(files, fn)
 168                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 169                         if err != nil {
 170                                 return 0, nil, err
 171                         }
 172                         fnClean := strings.TrimSuffix(fn, "."+algo)
 173                         release := releaseFiles[fnClean]
 174                         if release == nil {
 175                                 fi, err := entry.Info()
 176                                 if err != nil {
 177                                         return 0, nil, err
 178                                 }
 179                                 release = &PkgReleaseInfo{
 180                                         Filename: fnClean,
 181                                         Version:  filenameToVersion(fnClean),
 182                                         UploadTimeISO8601: fi.ModTime().UTC().Truncate(
 183                                                 time.Second,
 184                                         ).Format(time.RFC3339),
 185                                         Digests: make(map[string]string),
 186                                 }
 187                                 releaseFiles[fnClean] = release
 188                                 if entry, exists := files[fnClean]; exists {
 189                                         if doSize {
 190                                                 fi, err := entry.Info()
 191                                                 if err != nil {
 192                                                         return 0, nil, err
 193                                                 }
 194                                                 release.Size = fi.Size()
 195                                         }
 196                                         delete(files, fnClean)
 197                                 }
 198                                 if _, exists := files[fnClean+GPGSigExt]; exists {
 199                                         release.HasSig = true
 200                                         delete(files, fnClean+GPGSigExt)
 201                                 }
 202                         }
 203                         release.Digests[algo] = hex.EncodeToString(digest)
 204                 }
 205         }
 206         releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
 207         for _, release := range releaseFiles {
 208                 releases = append(releases, release)
 209         }
 210         sort.Sort(PkgReleaseInfoByName(releases))
 211         fi, err := os.Stat(dirPath)
 212         if err != nil {
 213                 return 0, nil, err
 214         }
 215         serial := fi.ModTime().Unix()
 216         if fi, err = os.Stat(filepath.Join(dirPath, MetadataFile)); err == nil {
 217                 serial += fi.ModTime().Unix()
 218         }
 219         return serial, releases, nil
 220 }
 221
 222 func serveListDir(
 223         w http.ResponseWriter,
 224         r *http.Request,
 225         pkgName string,
 226         autorefresh, gpgUpdate bool,
 227 ) {
 228         dirPath := filepath.Join(Root, pkgName)
 229         if autorefresh {
 230                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 231                         return
 232                 }
 233         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
 234                 !refreshDir(w, r, pkgName, "", false) {
 235                 return
 236         }
 237         serial, releases, err := listDir(pkgName, false)
 238         if err != nil {
 239                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 240                 http.Error(w, err.Error(), http.StatusInternalServerError)
 241                 return
 242         }
 243         for _, release := range releases {
 244                 singleDigest := make(map[string]string)
 245                 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
 246                         singleDigest[HashAlgoSHA256] = digest
 247                 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
 248                         singleDigest[HashAlgoSHA512] = digest
 249                 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
 250                         singleDigest[HashAlgoBLAKE2b256] = digest
 251                 } else {
 252                         singleDigest = release.Digests
 253                 }
 254                 release.Digests = singleDigest
 255         }
 256         var buf bytes.Buffer
 257         err = HTMLReleasesTmpl.Execute(&buf, struct {
 258                 RefreshURLPath string
 259                 PkgName        string
 260                 Releases       []*PkgReleaseInfo
 261         }{
 262                 RefreshURLPath: *RefreshURLPath,
 263                 PkgName:        pkgName,
 264                 Releases:       releases,
 265         })
 266         if err != nil {
 267                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 268                 http.Error(w, err.Error(), http.StatusInternalServerError)
 269                 return
 270         }
 271         w.Header().Set("X-PyPI-Last-Serial", strconv.FormatInt(serial, 10))
 272         w.Write(buf.Bytes())
 273         w.Write([]byte(fmt.Sprintf("<!--SERIAL %d-->\n", serial)))
 274 }