commit - bc1f382f7aba640a7b6949938af27d41c1cabacf
commit + 31ba223621c960bfec6da52df2ab65f2b0791d68
blob - /dev/null
blob + 9b95dcd8f24e8a9a1ad8900a16ece71c03e03a94 (mode 644)
--- /dev/null
+++ lib/got_lib_ratelimit.h
+/*
+ * Copyright (c) 2022 Stefan Sperling <stsp@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+struct got_ratelimit {
+ struct timespec last;
+ struct timespec interval;
+};
+
+void got_ratelimit_init(struct got_ratelimit *, time_t, unsigned int);
+const struct got_error *got_ratelimit_check(int *, struct got_ratelimit *);
blob - d0f59316e2f326a98b4f4e4a666136791edc48b1
blob + ff8f487b05d2d9af496f1b846a580ce369fd95af
--- lib/pack_create.c
+++ lib/pack_create.c
#include <sys/types.h>
#include <sys/uio.h>
#include <sys/stat.h>
+#include <sys/time.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sha1.h>
+#include <time.h>
#include <limits.h>
#include <zlib.h>
#include "got_lib_pack.h"
#include "got_lib_privsep.h"
#include "got_lib_repository.h"
+#include "got_lib_ratelimit.h"
#ifndef MIN
#define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
return NULL;
}
+
+static const struct got_error *
+report_progress(got_pack_progress_cb progress_cb, void *progress_arg,
+ struct got_ratelimit *rl, off_t packfile_size, int ncommits,
+ int nobj_total, int obj_deltify, int nobj_written)
+{
+ const struct got_error *err;
+ int elapsed;
+
+ if (progress_cb == NULL)
+ return NULL;
+ err = got_ratelimit_check(&elapsed, rl);
+ if (err || !elapsed)
+ return err;
+ return progress_cb(progress_arg, packfile_size, ncommits,
+ nobj_total, obj_deltify, nobj_written);
+}
+
static const struct got_error *
pick_deltas(struct got_pack_meta **meta, int nmeta, int nours,
FILE *delta_cache, struct got_repository *repo,
got_pack_progress_cb progress_cb, void *progress_arg,
- got_cancel_cb cancel_cb, void *cancel_arg)
+ struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
{
const struct got_error *err = NULL;
struct got_pack_meta *m = NULL, *base = NULL;
if (err)
break;
}
- if (progress_cb) {
- err = progress_cb(progress_arg, 0L, nours, nmeta, i, 0);
- if (err)
- goto done;
- }
+ err = report_progress(progress_cb, progress_arg, rl,
+ 0L, nours, nmeta, i, 0);
+ if (err)
+ goto done;
m = meta[i];
if (m->obj_type == GOT_OBJ_TYPE_COMMIT ||
struct got_object_id **theirs, int ntheirs,
struct got_object_id **ours, int nours, struct got_repository *repo,
int loose_obj_only, got_pack_progress_cb progress_cb, void *progress_arg,
- got_cancel_cb cancel_cb, void *cancel_arg)
+ struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
{
const struct got_error *err = NULL;
struct got_object_id **ids = NULL;
loose_obj_only, cancel_cb, cancel_arg);
if (err)
goto done;
- if (progress_cb) {
- err = progress_cb(progress_arg, 0L, nours,
- v.nmeta, 0, 0);
- if (err)
- goto done;
- }
+ err = report_progress(progress_cb, progress_arg, rl,
+ 0L, nours, v.nmeta, 0, 0);
+ if (err)
+ goto done;
}
for (i = 0; i < ntheirs; i++) {
loose_obj_only, cancel_cb, cancel_arg);
if (err)
goto done;
- if (progress_cb) {
- err = progress_cb(progress_arg, 0L, nours,
- v.nmeta, 0, 0);
- if (err)
- goto done;
- }
+ err = report_progress(progress_cb, progress_arg, rl,
+ 0L, nours, v.nmeta, 0, 0);
+ if (err)
+ goto done;
}
for (i = 0; i < nobj; i++) {
loose_obj_only, cancel_cb, cancel_arg);
if (err)
goto done;
- if (progress_cb) {
- err = progress_cb(progress_arg, 0L, nours,
- v.nmeta, 0, 0);
- if (err)
- goto done;
- }
+ if (err)
+ goto done;
+ err = report_progress(progress_cb, progress_arg, rl,
+ 0L, nours, v.nmeta, 0, 0);
+ if (err)
+ goto done;
}
for (i = 0; i < nours; i++) {
loose_obj_only, cancel_cb, cancel_arg);
if (err)
goto done;
- if (progress_cb) {
- err = progress_cb(progress_arg, 0L, nours,
- v.nmeta, 0, 0);
- if (err)
- goto done;
- }
+ err = report_progress(progress_cb, progress_arg, rl,
+ 0L, nours, v.nmeta, 0, 0);
+ if (err)
+ goto done;
}
+ if (progress_cb) {
+ err = progress_cb(progress_arg, 0L, nours, v.nmeta, 0, 0);
+ if (err)
+ goto done;
+ }
done:
for (i = 0; i < nobj; i++) {
free(ids[i]);
struct got_pack_meta **meta, int nmeta, int nours,
int use_offset_deltas, struct got_repository *repo,
got_pack_progress_cb progress_cb, void *progress_arg,
+ struct got_ratelimit *rl,
got_cancel_cb cancel_cb, void *cancel_arg)
{
const struct got_error *err = NULL;
goto done;
qsort(meta, nmeta, sizeof(struct got_pack_meta *), write_order_cmp);
for (i = 0; i < nmeta; i++) {
- if (progress_cb) {
- err = progress_cb(progress_arg, packfile_size, nours,
- nmeta, nmeta, i);
- if (err)
- goto done;
- }
+ err = report_progress(progress_cb, progress_arg, rl,
+ packfile_size, nours, nmeta, nmeta, i);
+ if (err)
+ goto done;
m = meta[i];
m->off = ftello(packfile);
err = got_object_raw_open(&raw, &outfd, repo, &m->id);
err = got_ferror(packfile, GOT_ERR_IO);
packfile_size += SHA1_DIGEST_LENGTH;
packfile_size += sizeof(struct got_packfile_hdr);
- err = progress_cb(progress_arg, packfile_size, nours,
- nmeta, nmeta, nmeta);
- if (err)
- goto done;
+ if (progress_cb) {
+ err = progress_cb(progress_arg, packfile_size, nours,
+ nmeta, nmeta, nmeta);
+ if (err)
+ goto done;
+ }
done:
if (delta_file && fclose(delta_file) == EOF && err == NULL)
err = got_error_from_errno("fclose");
struct got_pack_meta **meta;
int nmeta;
FILE *delta_cache = NULL;
+ struct got_ratelimit rl;
+ got_ratelimit_init(&rl, 0, 500);
+
err = read_meta(&meta, &nmeta, theirs, ntheirs, ours, nours, repo,
- loose_obj_only, progress_cb, progress_arg, cancel_cb, cancel_arg);
+ loose_obj_only, progress_cb, progress_arg, &rl,
+ cancel_cb, cancel_arg);
if (err)
return err;
if (nmeta > 0) {
err = pick_deltas(meta, nmeta, nours, delta_cache, repo,
- progress_cb, progress_arg, cancel_cb, cancel_arg);
+ progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
if (err)
goto done;
if (fseeko(delta_cache, 0L, SEEK_SET) == -1) {
}
err = genpack(packsha1, packfile, delta_cache, meta, nmeta, nours, 1,
- repo, progress_cb, progress_arg, cancel_cb, cancel_arg);
+ repo, progress_cb, progress_arg, &rl, cancel_cb, cancel_arg);
if (err)
goto done;
done:
blob - /dev/null
blob + 219e7905e4765ae964a3ab7a1e8145d1ce344c2f (mode 644)
--- /dev/null
+++ lib/ratelimit.c
+/*
+ * Copyright (c) 2022 Stefan Sperling <stsp@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/time.h>
+
+#include <stdio.h>
+#include <limits.h>
+#include <string.h>
+#include <time.h>
+
+#include "got_lib_ratelimit.h"
+
+#include "got_error.h"
+
+void
+got_ratelimit_init(struct got_ratelimit *rl, time_t interval_sec,
+ unsigned int interval_msec)
+{
+ memset(rl, 0, sizeof(*rl));
+ rl->interval.tv_sec = interval_sec;
+ rl->interval.tv_nsec = interval_msec * 1000000UL;
+}
+
+const struct got_error *
+got_ratelimit_check(int *elapsed, struct got_ratelimit *rl)
+{
+ struct timespec now, delta;
+
+ if (clock_gettime(CLOCK_MONOTONIC, &now) == -1)
+ return got_error_from_errno("clock_gettime");
+
+ if (timespecisset(&rl->last)) {
+ timespecsub(&now, &rl->last, &delta);
+ *elapsed = timespeccmp(&delta, &rl->interval, >=) ? 1 : 0;
+ } else
+ *elapsed = 1;
+
+ if (*elapsed)
+ rl->last = now;
+
+ return NULL;
+}
blob - 739ff3d01b9b761e9af99eed1c1ef1141465eb90
blob + 831a3bcbe7d86084e4f2106af90f970ab8dcbc8f
--- lib/repository_admin.c
+++ lib/repository_admin.c
#include "got_lib_pack_create.h"
#include "got_lib_sha1.h"
#include "got_lib_lockfile.h"
+#include "got_lib_ratelimit.h"
#ifndef nitems
#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
}
static const struct got_error *
+report_cleanup_progress(got_cleanup_progress_cb progress_cb,
+ void *progress_arg, struct got_ratelimit *rl,
+ int nloose, int ncommits, int npurged)
+{
+ const struct got_error *err;
+ int elapsed;
+
+ if (progress_cb == NULL)
+ return NULL;
+
+ err = got_ratelimit_check(&elapsed, rl);
+ if (err || !elapsed)
+ return err;
+
+ return progress_cb(progress_arg, nloose, ncommits, npurged);
+}
+
+static const struct got_error *
get_loose_object_ids(struct got_object_idset **loose_ids, off_t *ondisk_size,
got_cleanup_progress_cb progress_cb, void *progress_arg,
- struct got_repository *repo)
+ struct got_ratelimit *rl, struct got_repository *repo)
{
const struct got_error *err = NULL;
char *path_objects = NULL, *path = NULL;
err = got_object_idset_add(*loose_ids, &id, NULL);
if (err)
goto done;
- if (progress_cb) {
- err = progress_cb(progress_arg,
- got_object_idset_num_elements(*loose_ids),
- -1, -1);
- if (err)
- goto done;
- }
+ err = report_cleanup_progress(progress_cb,
+ progress_arg, rl,
+ got_object_idset_num_elements(*loose_ids), -1, -1);
+ if (err)
+ goto done;
}
if (closedir(dir) != 0) {
load_commit_or_tag(struct got_object_idset *loose_ids, int *ncommits,
int *npacked, struct got_object_idset *traversed_ids,
struct got_object_id *id, struct got_repository *repo,
- got_cleanup_progress_cb progress_cb, void *progress_arg, int nloose,
+ got_cleanup_progress_cb progress_cb, void *progress_arg,
+ struct got_ratelimit *rl, int nloose,
got_cancel_cb cancel_cb, void *cancel_arg)
{
const struct got_error *err;
if (commit || tag)
(*ncommits)++; /* scanned tags are counted as commits */
- if (progress_cb) {
- err = progress_cb(progress_arg, nloose, *ncommits, -1);
- if (err)
- break;
- }
+ err = report_cleanup_progress(progress_cb, progress_arg, rl,
+ nloose, *ncommits, -1);
+ if (err)
+ break;
if (commit) {
/* Find parent commits to scan. */
struct got_repository *repo;
got_cleanup_progress_cb progress_cb;
void *progress_arg;
+ struct got_ratelimit *rl;
int nloose;
int ncommits;
int npurged;
a->npurged++;
a->size_purged += sb.st_size;
- if (a->progress_cb) {
- err = a->progress_cb(a->progress_arg, a->nloose,
- a->ncommits, a->npurged);
- }
+ err = report_cleanup_progress(a->progress_cb, a->progress_arg,
+ a->rl, a->nloose, a->ncommits, a->npurged);
+ if (err)
+ goto done;
}
done:
if (fd != -1 && close(fd) == -1 && err == NULL)
struct got_reflist_entry *re;
struct purge_loose_object_arg arg;
time_t max_mtime = 0;
+ struct got_ratelimit rl;
TAILQ_INIT(&refs);
+ got_ratelimit_init(&rl, 0, 500);
*size_before = 0;
*size_after = 0;
*npacked = 0;
err = get_loose_object_ids(&loose_ids, size_before,
- progress_cb, progress_arg, repo);
+ progress_cb, progress_arg, &rl, repo);
if (err)
return err;
nloose = got_object_idset_num_elements(loose_ids);
for (i = 0; i < nreferenced; i++) {
struct got_object_id *id = referenced_ids[i];
err = load_commit_or_tag(loose_ids, &ncommits, npacked,
- traversed_ids, id, repo, progress_cb, progress_arg, nloose,
- cancel_cb, cancel_arg);
+ traversed_ids, id, repo, progress_cb, progress_arg, &rl,
+ nloose, cancel_cb, cancel_arg);
if (err)
goto done;
}
- /* Produce a final progress report in case no objects can be purged. */
- if (got_object_idset_num_elements(loose_ids) == 0 && progress_cb) {
- err = progress_cb(progress_arg, nloose, ncommits, 0);
- if (err)
- goto done;
- }
-
/* Any remaining loose objects are unreferenced and can be purged. */
arg.repo = repo;
arg.progress_arg = progress_arg;
arg.progress_cb = progress_cb;
+ arg.rl = &rl;
arg.nloose = nloose;
arg.npurged = 0;
arg.size_purged = 0;
if (err)
goto done;
*size_after = *size_before - arg.size_purged;
+
+ /* Produce a final progress report. */
+ if (progress_cb) {
+ err = progress_cb(progress_arg, nloose, ncommits,
+ got_object_idset_num_elements(loose_ids));
+ if (err)
+ goto done;
+ }
done:
got_object_idset_free(loose_ids);
got_object_idset_free(traversed_ids);
