list.go

   1 /*
   2 GoCheese -- Python private package repository and caching proxy
   3 Copyright (C) 2019-2021 Sergey Matveev <stargrave@stargrave.org>
   4
   5 This program is free software: you can redistribute it and/or modify
   6 it under the terms of the GNU General Public License as published by
   7 the Free Software Foundation, version 3 of the License.
   8
   9 This program is distributed in the hope that it will be useful,
  10 but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 GNU General Public License for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17
  18 package main
  19
  20 import (
  21         "bytes"
  22         "encoding/hex"
  23         "errors"
  24         "html/template"
  25         "io/fs"
  26         "io/ioutil"
  27         "log"
  28         "net/http"
  29         "os"
  30         "path/filepath"
  31         "sort"
  32         "strings"
  33         "time"
  34 )
  35
  36 // https://warehouse.pypa.io/api-reference/legacy.html
  37 var (
  38         HTMLRootTmpl = template.Must(template.New("root").Parse(`<!DOCTYPE html>
  39 <html>
  40   <head>
  41     <meta name="pypi:repository-version" content="1.0">
  42     <title>Links for root</title>
  43   </head>
  44   <body>{{$Refresh := .RefreshURLPath}}{{range .Packages}}
  45     <a href="{{$Refresh}}{{.}}/">{{.}}</a><br/>
  46 {{- end}}
  47   </body>
  48 </html>
  49 `))
  50         HTMLReleasesTmpl = template.Must(template.New("list").Parse(`<!DOCTYPE html>
  51 <html>
  52   <head>
  53     <meta name="pypi:repository-version" content="1.0">
  54     <title>Links for {{.PkgName}}</title>
  55   </head>
  56   <body>{{$Refresh := .RefreshURLPath}}{{$PkgName := .PkgName}}{{range .Releases}}
  57     <a href="{{$Refresh}}{{$PkgName}}/{{.Filename -}}
  58         #{{range $a, $d := .Digests}}{{$a}}={{$d}}{{end -}}
  59         {{with .HasSig}} data-gpg-sig=true{{end}}">{{.Filename}}</a><br/>
  60 {{- end}}
  61   </body>
  62 </html>
  63 `))
  64         KnownExts = []string{".tar.bz2", ".tar.gz", ".whl", ".zip", ".egg",
  65                 ".exe", ".dmg", ".msi", ".rpm", ".deb", ".tgz"}
  66 )
  67
  68 func listRoot(w http.ResponseWriter, r *http.Request) {
  69         files, err := ioutil.ReadDir(Root)
  70         if err != nil {
  71                 log.Println("error", r.RemoteAddr, "root", err)
  72                 http.Error(w, err.Error(), http.StatusInternalServerError)
  73                 return
  74         }
  75         packages := make([]string, 0, len(files))
  76         for _, f := range files {
  77                 packages = append(packages, f.Name())
  78         }
  79         sort.Strings(packages)
  80         var buf bytes.Buffer
  81         err = HTMLRootTmpl.Execute(&buf, struct {
  82                 RefreshURLPath string
  83                 Packages       []string
  84         }{
  85                 RefreshURLPath: *RefreshURLPath,
  86                 Packages:       packages,
  87         })
  88         if err != nil {
  89                 log.Println("error", r.RemoteAddr, "root", err)
  90                 http.Error(w, err.Error(), http.StatusInternalServerError)
  91                 return
  92         }
  93         w.Write(buf.Bytes())
  94 }
  95
  96 type PkgReleaseInfoByName []*PkgReleaseInfo
  97
  98 func (a PkgReleaseInfoByName) Len() int {
  99         return len(a)
 100 }
 101
 102 func (a PkgReleaseInfoByName) Swap(i, j int) {
 103         a[i], a[j] = a[j], a[i]
 104 }
 105
 106 func (a PkgReleaseInfoByName) Less(i, j int) bool {
 107         if a[i].Version == a[j].Version {
 108                 return a[i].Filename < a[j].Filename
 109         }
 110         return a[i].Version < a[j].Version
 111 }
 112
 113 // Version format is too complicated: https://www.python.org/dev/peps/pep-0386/
 114 // So here is very simple parser working good enough for most packages
 115 func filenameToVersion(fn string) string {
 116         fn = strings.TrimSuffix(fn, GPGSigExt)
 117         var trimmed string
 118         for _, ext := range KnownExts {
 119                 trimmed = strings.TrimSuffix(fn, ext)
 120                 if trimmed != fn {
 121                         fn = trimmed
 122                         break
 123                 }
 124         }
 125         cols := strings.Split(fn, "-")
 126         for i := 0; i < len(cols); i++ {
 127                 if len(cols[i]) == 0 {
 128                         continue
 129                 }
 130                 if ('0' <= cols[i][0]) && (cols[i][0] <= '9') {
 131                         return cols[i]
 132                 }
 133         }
 134         if len(cols) > 1 {
 135                 return cols[1]
 136         }
 137         return cols[0]
 138 }
 139
 140 func listDir(pkgName string, doSize bool) (int, []*PkgReleaseInfo, error) {
 141         dirPath := filepath.Join(Root, pkgName)
 142         entries, err := os.ReadDir(dirPath)
 143         if err != nil {
 144                 return 0, nil, err
 145         }
 146         files := make(map[string]fs.DirEntry, len(entries))
 147         for _, entry := range entries {
 148                 if entry.IsDir() {
 149                         continue
 150                 }
 151                 if entry.Name()[0] == '.' {
 152                         continue
 153                 }
 154                 files[entry.Name()] = entry
 155         }
 156         releaseFiles := make(map[string]*PkgReleaseInfo)
 157         for _, algo := range KnownHashAlgos {
 158                 for fn, entry := range files {
 159                         if Killed {
 160                                 return 0, nil, errors.New("killed")
 161                         }
 162                         if !strings.HasSuffix(fn, "."+algo) {
 163                                 continue
 164                         }
 165                         delete(files, fn)
 166                         digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn))
 167                         if err != nil {
 168                                 return 0, nil, err
 169                         }
 170                         fnClean := strings.TrimSuffix(fn, "."+algo)
 171                         release := releaseFiles[fnClean]
 172                         if release == nil {
 173                                 fi, err := entry.Info()
 174                                 if err != nil {
 175                                         return 0, nil, err
 176                                 }
 177                                 release = &PkgReleaseInfo{
 178                                         Filename: fnClean,
 179                                         Version:  filenameToVersion(fnClean),
 180                                         UploadTimeISO8601: fi.ModTime().UTC().Truncate(
 181                                                 time.Second,
 182                                         ).Format(time.RFC3339),
 183                                         Digests: make(map[string]string),
 184                                 }
 185                                 releaseFiles[fnClean] = release
 186                                 if entry, exists := files[fnClean]; exists {
 187                                         if doSize {
 188                                                 fi, err := entry.Info()
 189                                                 if err != nil {
 190                                                         return 0, nil, err
 191                                                 }
 192                                                 release.Size = fi.Size()
 193                                         }
 194                                         delete(files, fnClean)
 195                                 }
 196                                 if _, exists := files[fnClean+GPGSigExt]; exists {
 197                                         release.HasSig = true
 198                                         delete(files, fnClean+GPGSigExt)
 199                                 }
 200                         }
 201                         release.Digests[algo] = hex.EncodeToString(digest)
 202                 }
 203         }
 204         releases := make([]*PkgReleaseInfo, 0, len(releaseFiles))
 205         for _, release := range releaseFiles {
 206                 releases = append(releases, release)
 207         }
 208         sort.Sort(PkgReleaseInfoByName(releases))
 209         return len(entries), releases, nil
 210 }
 211
 212 func serveListDir(
 213         w http.ResponseWriter,
 214         r *http.Request,
 215         pkgName string,
 216         autorefresh, gpgUpdate bool,
 217 ) {
 218         dirPath := filepath.Join(Root, pkgName)
 219         if autorefresh {
 220                 if !refreshDir(w, r, pkgName, "", gpgUpdate) {
 221                         return
 222                 }
 223         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) &&
 224                 !refreshDir(w, r, pkgName, "", false) {
 225                 return
 226         }
 227         _, releases, err := listDir(pkgName, false)
 228         if err != nil {
 229                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 230                 http.Error(w, err.Error(), http.StatusInternalServerError)
 231                 return
 232         }
 233         for _, release := range releases {
 234                 singleDigest := make(map[string]string)
 235                 if digest, exists := release.Digests[HashAlgoSHA256]; exists {
 236                         singleDigest[HashAlgoSHA256] = digest
 237                 } else if digest, exists := release.Digests[HashAlgoSHA512]; exists {
 238                         singleDigest[HashAlgoSHA512] = digest
 239                 } else if digest, exists := release.Digests[HashAlgoBLAKE2b256]; exists {
 240                         singleDigest[HashAlgoBLAKE2b256] = digest
 241                 } else {
 242                         singleDigest = release.Digests
 243                 }
 244                 release.Digests = singleDigest
 245         }
 246         var buf bytes.Buffer
 247         err = HTMLReleasesTmpl.Execute(&buf, struct {
 248                 RefreshURLPath string
 249                 PkgName        string
 250                 Releases       []*PkgReleaseInfo
 251         }{
 252                 RefreshURLPath: *RefreshURLPath,
 253                 PkgName:        pkgName,
 254                 Releases:       releases,
 255         })
 256         if err != nil {
 257                 log.Println("error", r.RemoteAddr, "list", pkgName, err)
 258                 http.Error(w, err.Error(), http.StatusInternalServerError)
 259                 return
 260         }
 261         w.Write(buf.Bytes())
 262 }