From: Sergey Matveev Date: Mon, 9 Dec 2019 05:38:40 +0000 (+0300) Subject: More errors logging X-Git-Tag: v2.2.0~1 X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=commitdiff_plain;h=5f74f27dba4c477281305586ef96a39b4751851e More errors logging --- diff --git a/gocheese.go b/gocheese.go index 67e7829..9b7a80d 100644 --- a/gocheese.go +++ b/gocheese.go @@ -103,14 +103,15 @@ var ( pypiURLParsed *url.URL ) -func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { - path := filepath.Join(*root, dir) +func mkdirForPkg(w http.ResponseWriter, r *http.Request, pkgName string) bool { + path := filepath.Join(*root, pkgName) if _, err := os.Stat(path); os.IsNotExist(err) { if err = os.Mkdir(path, os.FileMode(0777)); err != nil { + log.Println("error", r.RemoteAddr, "mkdir", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "mkdir", dir) + log.Println(r.RemoteAddr, "mkdir", pkgName) } return true } @@ -118,6 +119,7 @@ func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { func listRoot(w http.ResponseWriter, r *http.Request) { files, err := ioutil.ReadDir(*root) if err != nil { + log.Println("error", r.RemoteAddr, "root", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -139,20 +141,20 @@ func listRoot(w http.ResponseWriter, r *http.Request) { func listDir( w http.ResponseWriter, r *http.Request, - dir string, - autorefresh, - gpgUpdate bool, + pkgName string, + autorefresh, gpgUpdate bool, ) { - dirPath := filepath.Join(*root, dir) + dirPath := filepath.Join(*root, pkgName) if autorefresh { - if !refreshDir(w, r, dir, "", gpgUpdate) { + if !refreshDir(w, r, pkgName, "", gpgUpdate) { return } - } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) { + } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, pkgName, "", false) { return } fis, err := ioutil.ReadDir(dirPath) if err != nil { + log.Println("error", r.RemoteAddr, "list", pkgName, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -161,7 +163,7 @@ func listDir( files[fi.Name()] = struct{}{} } var result bytes.Buffer - result.WriteString(fmt.Sprintf(HTMLBegin, dir)) + result.WriteString(fmt.Sprintf(HTMLBegin, pkgName)) for _, algo := range knownHashAlgos { for fn, _ := range files { if killed { @@ -175,6 +177,7 @@ func listDir( delete(files, fn) digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) if err != nil { + log.Println("error", r.RemoteAddr, "list", fn, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -188,7 +191,7 @@ func listDir( result.WriteString(fmt.Sprintf( HTMLElement, strings.Join([]string{ - *refreshURLPath, dir, "/", fnClean, + *refreshURLPath, pkgName, "/", fnClean, "#", algo, "=", hex.EncodeToString(digest), }, ""), gpgSigAttr, @@ -200,11 +203,11 @@ func listDir( w.Write(result.Bytes()) } -func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { +func servePkg(w http.ResponseWriter, r *http.Request, pkgName, filename string) { log.Println(r.RemoteAddr, "get", filename) - path := filepath.Join(*root, dir, filename) + path := filepath.Join(*root, pkgName, filename) if _, err := os.Stat(path); os.IsNotExist(err) { - if !refreshDir(w, r, dir, filename, false) { + if !refreshDir(w, r, pkgName, filename, false) { return } } @@ -280,7 +283,9 @@ func main() { log.Fatalln(err) } refreshPasswd() - log.Println("root:", *root, "bind:", *bind) + log.Println("root:", *root) + log.Println("bind:", *bind) + log.Println("pypi:", *pypiURL) ln, err := net.Listen("tcp", *bind) if err != nil { @@ -304,14 +309,14 @@ func main() { signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT) go func() { for range needsRefreshPasswd { - log.Println("Refreshing passwords") + log.Println("refreshing passwords") refreshPasswd() } }() go func(s *http.Server) { <-needsShutdown killed = true - log.Println("Shutting down") + log.Println("shutting down") ctx, cancel := context.WithTimeout(context.TODO(), time.Minute) exitErr <- s.Shutdown(ctx) cancel() diff --git a/passwd.go b/passwd.go index f6760d2..1dffe21 100644 --- a/passwd.go +++ b/passwd.go @@ -77,7 +77,7 @@ func refreshPasswd() { log.Fatalf("%s:%d: %s: %s", *passwdPath, i, login, err) } passwordsNew[login] = auther - log.Println("Added password for " + login) + log.Println("added password for " + login) } passwords = passwordsNew } diff --git a/refresh.go b/refresh.go index 3404a37..4a5e962 100644 --- a/refresh.go +++ b/refresh.go @@ -48,28 +48,29 @@ func blake2b256New() hash.Hash { func refreshDir( w http.ResponseWriter, r *http.Request, - dir, - filenameGet string, + pkgName, filenameGet string, gpgUpdate bool, ) bool { - if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { + if _, err := os.Stat(filepath.Join(*root, pkgName, InternalFlag)); err == nil { return true } - resp, err := http.Get(*pypiURL + dir + "/") + resp, err := http.Get(*pypiURL + pkgName + "/") if err != nil { + log.Println("error", r.RemoteAddr, "refresh", pkgName, err) http.Error(w, err.Error(), http.StatusBadGateway) return false } body, err := ioutil.ReadAll(resp.Body) resp.Body.Close() if err != nil { + log.Println("error", r.RemoteAddr, "refresh", pkgName, err) http.Error(w, err.Error(), http.StatusBadGateway) return false } - if !mkdirForPkg(w, r, dir) { + if !mkdirForPkg(w, r, pkgName) { return false } - dirPath := filepath.Join(*root, dir) + dirPath := filepath.Join(*root, pkgName) for _, lineRaw := range bytes.Split(body, []byte("\n")) { submatches := pkgPyPI.FindStringSubmatch(string(lineRaw)) if len(submatches) == 0 { @@ -79,12 +80,13 @@ func refreshDir( filename := submatches[2] pkgURL, err := url.Parse(uri) if err != nil { + log.Println("error", r.RemoteAddr, "refresh", uri, err) http.Error(w, err.Error(), http.StatusBadGateway) return false } if pkgURL.Fragment == "" { - log.Println(r.RemoteAddr, "pypi", filename, "no digest provided") + log.Println(r.RemoteAddr, "pypi", filename, "no digest") http.Error(w, "no digest provided", http.StatusBadGateway) return false } @@ -93,12 +95,13 @@ func refreshDir( // Ancient non PEP-0503 PyPIs, assume MD5 digestInfo = []string{"md5", digestInfo[0]} } else if len(digestInfo) != 2 { - log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided") + log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest") http.Error(w, "invalid digest provided", http.StatusBadGateway) return false } digest, err := hex.DecodeString(digestInfo[1]) if err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest") http.Error(w, err.Error(), http.StatusBadGateway) return false } @@ -119,15 +122,12 @@ func refreshDir( hasherNew = blake2b256New hashSize = blake2b.Size256 default: - log.Println( - r.RemoteAddr, "pypi", filename, - "unknown digest algorithm", hashAlgo, - ) + log.Println("error", r.RemoteAddr, "pypi", filename, "unknown digest", hashAlgo) http.Error(w, "unknown digest algorithm", http.StatusBadGateway) return false } if len(digest) != hashSize { - log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length") + log.Println("error", r.RemoteAddr, "pypi", filename, "invalid digest length") http.Error(w, "invalid digest length", http.StatusBadGateway) return false } @@ -146,10 +146,10 @@ func refreshDir( http.Error(w, "shutting down", http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "pypi download", filename) + log.Println(r.RemoteAddr, "pypi", filename, "download") resp, err = http.Get(uri) if err != nil { - log.Println(r.RemoteAddr, "pypi download error:", err.Error()) + log.Println("error", r.RemoteAddr, "pypi", filename, "download", err) http.Error(w, err.Error(), http.StatusBadGateway) return false } @@ -158,6 +158,7 @@ func refreshDir( hasherSHA256 := sha256.New() dst, err := TempFile(dirPath) if err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } @@ -170,12 +171,14 @@ func refreshDir( if _, err = io.Copy(wr, resp.Body); err != nil { os.Remove(dst.Name()) dst.Close() + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } if err = dstBuf.Flush(); err != nil { os.Remove(dst.Name()) dst.Close() + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } @@ -189,18 +192,22 @@ func refreshDir( if err = dst.Sync(); err != nil { os.Remove(dst.Name()) dst.Close() + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } if err = dst.Close(); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } if err = os.Rename(dst.Name(), path); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } if err = DirSync(dirPath); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } @@ -230,14 +237,15 @@ func refreshDir( goto GPGSigSkip } if !bytes.HasPrefix(sig, []byte("-----BEGIN PGP SIGNATURE-----")) { - log.Println(r.RemoteAddr, "pypi non PGP signature", filename) + log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "non PGP") goto GPGSigSkip } if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { + log.Println("error", r.RemoteAddr, "pypi", filename+GPGSigExt, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "pypi downloaded signature", filename) + log.Println(r.RemoteAddr, "pypi", filename+GPGSigExt, "downloaded") } GPGSigSkip: path = path + "." + hashAlgo @@ -246,11 +254,13 @@ func refreshDir( continue } if !os.IsNotExist(err) { + log.Println("error", r.RemoteAddr, "pypi", path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } - log.Println(r.RemoteAddr, "pypi touch", filename) + log.Println(r.RemoteAddr, "pypi", filename, "touch") if err = WriteFileSync(dirPath, path, digest); err != nil { + log.Println("error", r.RemoteAddr, "pypi", path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return false } diff --git a/upload.go b/upload.go index 0e7db61..6a5e9bd 100644 --- a/upload.go +++ b/upload.go @@ -82,7 +82,7 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { log.Println(r.RemoteAddr, "put", filename, "by", username) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) + log.Println(r.RemoteAddr, filename, "already exists") http.Error(w, "already exists", http.StatusBadRequest) return } @@ -92,11 +92,13 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { src, err := file.Open() defer src.Close() if err != nil { + log.Println("error", r.RemoteAddr, filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } dst, err := TempFile(dirPath) if err != nil { + log.Println("error", r.RemoteAddr, filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -104,18 +106,21 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { hasher := sha256.New() wr := io.MultiWriter(hasher, dst) if _, err = io.Copy(wr, src); err != nil { + log.Println("error", r.RemoteAddr, filename, err) os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = dstBuf.Flush(); err != nil { + log.Println("error", r.RemoteAddr, filename, err) os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = dst.Sync(); err != nil { + log.Println("error", r.RemoteAddr, filename, err) os.Remove(dst.Name()) dst.Close() http.Error(w, err.Error(), http.StatusInternalServerError) @@ -134,14 +139,17 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { } } if err = os.Rename(dst.Name(), path); err != nil { + log.Println("error", r.RemoteAddr, path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = DirSync(dirPath); err != nil { + log.Println("error", r.RemoteAddr, dirPath, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = WriteFileSync(dirPath, path+"."+HashAlgoSHA256, digest); err != nil { + log.Println("error", r.RemoteAddr, path+"."+HashAlgoSHA256, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -149,6 +157,7 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { for _, file := range r.MultipartForm.File["gpg_signature"] { filename := file.Filename if _, exists := gpgSigsExpected[filename]; !exists { + log.Println(r.RemoteAddr, filename, "unexpected GPG signature filename") http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest) return } @@ -156,22 +165,25 @@ func serveUpload(w http.ResponseWriter, r *http.Request) { log.Println(r.RemoteAddr, "put", filename, "by", username) path := filepath.Join(dirPath, filename) if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) + log.Println(r.RemoteAddr, filename, "already exists") http.Error(w, "already exists", http.StatusBadRequest) return } src, err := file.Open() if err != nil { + log.Println("error", r.RemoteAddr, filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } sig, err := ioutil.ReadAll(src) src.Close() if err != nil { + log.Println("error", r.RemoteAddr, filename, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } if err = WriteFileSync(dirPath, path, sig); err != nil { + log.Println("error", r.RemoteAddr, path, err) http.Error(w, err.Error(), http.StatusInternalServerError) return }