vitalif
/
gogs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cron: add repository archive cleanup (#4061)

master
Unknwon 2017-02-18 19:35:59 -05:00
parent c69900325d
commit 0a2f87f941
No known key found for this signature in database
GPG Key ID: 25B575AE3213B2B3
9 changed files with 115 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
conf/app.ini
View File

@ -341,6 +341,13 @@ ARGS =
RUN_AT_START = true
SCHEDULE = @every 24h
; Cleanup repository archives
[cron.repo_archive_cleanup]
RUN_AT_START = false
SCHEDULE = @every 24h
; Time duration to check if archive should be cleaned
OLDER_THAN = 24h
[git]
; Disables highlight of added and removed changes
DISABLE_DIFF_HIGHLIGHT = false

2
gogs.go
View File

@ -16,7 +16,7 @@ import (
"github.com/gogits/gogs/modules/setting"
)
const APP_VER = "0.9.158.0218"
const APP_VER = "0.9.159.0218"
func init() {
setting.AppVer = APP_VER

97
models/repo.go
View File

@ -1597,8 +1597,70 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (repos []*Repository, _ int
return repos, count, sess.Limit(opts.PageSize, (opts.Page-1)*opts.PageSize).Find(&repos)
}
func DeleteOldRepositoryArchives() {
if taskStatusTable.IsRunning(_CLEAN_OLD_ARCHIVES) {
return
}
taskStatusTable.Start(_CLEAN_OLD_ARCHIVES)
defer taskStatusTable.Stop(_CLEAN_OLD_ARCHIVES)
log.Trace("Doing: DeleteOldRepositoryArchives")
formats := []string{"zip", "targz"}
oldestTime := time.Now().Add(-setting.Cron.RepoArchiveCleanup.OlderThan)
if err := x.Where("id > 0").Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
basePath := filepath.Join(repo.RepoPath(), "archives")
for _, format := range formats {
dirPath := filepath.Join(basePath, format)
if !com.IsDir(dirPath) {
continue
}
dir, err := os.Open(dirPath)
if err != nil {
log.Error(3, "Fail to open directory '%s': %v", dirPath, err)
continue
}
fis, err := dir.Readdir(0)
dir.Close()
if err != nil {
log.Error(3, "Fail to read directory '%s': %v", dirPath, err)
continue
}
for _, fi := range fis {
if fi.IsDir() || fi.ModTime().After(oldestTime) {
continue
}
archivePath := filepath.Join(dirPath, fi.Name())
if err = os.Remove(archivePath); err != nil {
desc := fmt.Sprintf("Fail to health delete archive '%s': %v", archivePath, err)
log.Warn(desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(3, "CreateRepositoryNotice: %v", err)
}
}
}
}
return nil
}); err != nil {
log.Error(2, "DeleteOldRepositoryArchives: %v", err)
}
}
// DeleteRepositoryArchives deletes all repositories' archives.
func DeleteRepositoryArchives() error {
if taskStatusTable.IsRunning(_CLEAN_OLD_ARCHIVES) {
return nil
}
taskStatusTable.Start(_CLEAN_OLD_ARCHIVES)
defer taskStatusTable.Stop(_CLEAN_OLD_ARCHIVES)
return x.Where("id > 0").Iterate(new(Repository),
func(idx int, bean interface{}) error {
repo := bean.(*Repository)
@ -1688,9 +1750,10 @@ func SyncRepositoryHooks() error {
var taskStatusTable = sync.NewStatusTable()
const (
_MIRROR_UPDATE = "mirror_update"
_GIT_FSCK = "git_fsck"
_CHECK_REPOs = "check_repos"
_MIRROR_UPDATE = "mirror_update"
_GIT_FSCK = "git_fsck"
_CHECK_REPO_STATS = "check_repos_stats"
_CLEAN_OLD_ARCHIVES = "clean_old_archives"
)
// GitFsck calls 'git fsck' to check repository health.
@ -1708,15 +1771,15 @@ func GitFsck() {
repo := bean.(*Repository)
repoPath := repo.RepoPath()
if err := git.Fsck(repoPath, setting.Cron.RepoHealthCheck.Timeout, setting.Cron.RepoHealthCheck.Args...); err != nil {
desc := fmt.Sprintf("Fail to health check repository (%s): %v", repoPath, err)
desc := fmt.Sprintf("Fail to health check repository '%s': %v", repoPath, err)
log.Warn(desc)
if err = CreateRepositoryNotice(desc); err != nil {
log.Error(4, "CreateRepositoryNotice: %v", err)
log.Error(3, "CreateRepositoryNotice: %v", err)
}
}
return nil
}); err != nil {
log.Error(4, "GitFsck: %v", err)
log.Error(2, "GitFsck: %v", err)
}
}
@ -1747,7 +1810,7 @@ type repoChecker struct {
func repoStatsCheck(checker *repoChecker) {
results, err := x.Query(checker.querySQL)
if err != nil {
log.Error(4, "Select %s: %v", checker.desc, err)
log.Error(2, "Select %s: %v", checker.desc, err)
return
}
for _, result := range results {
@ -1755,17 +1818,17 @@ func repoStatsCheck(checker *repoChecker) {
log.Trace("Updating %s: %d", checker.desc, id)
_, err = x.Exec(checker.correctSQL, id, id)
if err != nil {
log.Error(4, "Update %s[%d]: %v", checker.desc, id, err)
log.Error(2, "Update %s[%d]: %v", checker.desc, id, err)
}
}
}
func CheckRepoStats() {
if taskStatusTable.IsRunning(_CHECK_REPOs) {
if taskStatusTable.IsRunning(_CHECK_REPO_STATS) {
return
}
taskStatusTable.Start(_CHECK_REPOs)
defer taskStatusTable.Stop(_CHECK_REPOs)
taskStatusTable.Start(_CHECK_REPO_STATS)
defer taskStatusTable.Stop(_CHECK_REPO_STATS)
log.Trace("Doing: CheckRepoStats")
@ -1809,14 +1872,14 @@ func CheckRepoStats() {
desc := "repository count 'num_closed_issues'"
results, err := x.Query("SELECT repo.id FROM `repository` repo WHERE repo.num_closed_issues!=(SELECT COUNT(*) FROM `issue` WHERE repo_id=repo.id AND is_closed=? AND is_pull=?)", true, false)
if err != nil {
log.Error(4, "Select %s: %v", desc, err)
log.Error(2, "Select %s: %v", desc, err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
log.Trace("Updating %s: %d", desc, id)
_, err = x.Exec("UPDATE `repository` SET num_closed_issues=(SELECT COUNT(*) FROM `issue` WHERE repo_id=? AND is_closed=? AND is_pull=?) WHERE id=?", id, true, false, id)
if err != nil {
log.Error(4, "Update %s[%d]: %v", desc, id, err)
log.Error(2, "Update %s[%d]: %v", desc, id, err)
}
}
}
@ -1826,7 +1889,7 @@ func CheckRepoStats() {
// ***** START: Repository.NumForks *****
results, err = x.Query("SELECT repo.id FROM `repository` repo WHERE repo.num_forks!=(SELECT COUNT(*) FROM `repository` WHERE fork_id=repo.id)")
if err != nil {
log.Error(4, "Select repository count 'num_forks': %v", err)
log.Error(2, "Select repository count 'num_forks': %v", err)
} else {
for _, result := range results {
id := com.StrTo(result["id"]).MustInt64()
@ -1834,19 +1897,19 @@ func CheckRepoStats() {
repo, err := GetRepositoryByID(id)
if err != nil {
log.Error(4, "GetRepositoryByID[%d]: %v", id, err)
log.Error(2, "GetRepositoryByID[%d]: %v", id, err)
continue
}
rawResult, err := x.Query("SELECT COUNT(*) FROM `repository` WHERE fork_id=?", repo.ID)
if err != nil {
log.Error(4, "Select count of forks[%d]: %v", repo.ID, err)
log.Error(2, "Select count of forks[%d]: %v", repo.ID, err)
continue
}
repo.NumForks = int(parseCountResult(rawResult))
if err = UpdateRepository(repo, false); err != nil {
log.Error(4, "UpdateRepository[%d]: %v", id, err)
log.Error(2, "UpdateRepository[%d]: %v", id, err)
continue
}
}

4
modules/bindata/bindata.go
View File

File diff suppressed because one or more lines are too long

17
modules/cron/cron.go
View File

@ -25,7 +25,7 @@ func NewContext() {
if setting.Cron.UpdateMirror.Enabled {
entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, models.MirrorUpdate)
if err != nil {
log.Fatal(4, "Cron[Update mirrors]: %v", err)
log.Fatal(2, "Cron.(update mirrors): %v", err)
}
if setting.Cron.UpdateMirror.RunAtStart {
entry.Prev = time.Now()
@ -36,7 +36,7 @@ func NewContext() {
if setting.Cron.RepoHealthCheck.Enabled {
entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, models.GitFsck)
if err != nil {
log.Fatal(4, "Cron[Repository health check]: %v", err)
log.Fatal(2, "Cron.(repository health check): %v", err)
}
if setting.Cron.RepoHealthCheck.RunAtStart {
entry.Prev = time.Now()
@ -47,7 +47,7 @@ func NewContext() {
if setting.Cron.CheckRepoStats.Enabled {
entry, err = c.AddFunc("Check repository statistics", setting.Cron.CheckRepoStats.Schedule, models.CheckRepoStats)
if err != nil {
log.Fatal(4, "Cron[Check repository statistics]: %v", err)
log.Fatal(2, "Cron.(check repository statistics): %v", err)
}
if setting.Cron.CheckRepoStats.RunAtStart {
entry.Prev = time.Now()
@ -55,6 +55,17 @@ func NewContext() {
go models.CheckRepoStats()
}
}
if setting.Cron.RepoArchiveCleanup.Enabled {
entry, err = c.AddFunc("Repository archive cleanup", setting.Cron.RepoArchiveCleanup.Schedule, models.DeleteOldRepositoryArchives)
if err != nil {
log.Fatal(2, "Cron.(repository archive cleanup): %v", err)
}
if setting.Cron.RepoArchiveCleanup.RunAtStart {
entry.Prev = time.Now()
entry.ExecTimes++
go models.DeleteOldRepositoryArchives()
}
}
c.Start()
}

6
modules/setting/setting.go
View File

@ -208,6 +208,12 @@ var (
RunAtStart bool
Schedule string
} `ini:"cron.check_repo_stats"`
RepoArchiveCleanup struct {
Enabled bool
RunAtStart bool
Schedule string
OlderThan time.Duration
} `ini:"cron.repo_archive_cleanup"`
}
// Git settings

2
templates/.VERSION
View File

@ -1 +1 @@
0.9.158.0218
0.9.159.0218

2
vendor/github.com/gogits/git-module/repo_commit.go generated vendored
View File

@ -209,7 +209,7 @@ func (repo *Repository) CommitsByRangeSize(revision string, page, size int) (*li
return repo.parsePrettyFormatLogToList(stdout)
}
const DEFAULT_COMMITS_PAGE_SIZE = 50
const DEFAULT_COMMITS_PAGE_SIZE = 30
func (repo *Repository) CommitsByRange(revision string, page int) (*list.List, error) {
return repo.CommitsByRangeSize(revision, page, DEFAULT_COMMITS_PAGE_SIZE)

6
vendor/vendor.json vendored
View File

@ -159,10 +159,10 @@
"revisionTime": "2016-08-10T03:50:02Z"
},
{
"checksumSHA1": "mR45j8svu6CZu81VqN+lfgpCVjA=",
"checksumSHA1": "JPQWxQRYFpezMThuSqcPhvbJdq4=",
"path": "github.com/gogits/git-module",
"revision": "fa2ace85ecb113f89f6862d8a6e3075a7aa425b9",
"revisionTime": "2017-02-18T23:35:37Z"
"revision": "c882f3d24df5fb730d3b3c1b6cc64416dd22660c",
"revisionTime": "2017-02-18T23:44:51Z"
},
{
"checksumSHA1": "xvG+RgJODQqlmdAkHUQK2TyLR88=",