-
Notifications
You must be signed in to change notification settings - Fork 76
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move autoremove builds logic to a scheduled job
Separate the automatic removal of old builds from CDash's submission parsing logic. Instead, old builds will now be periodically cleaned up as a scheduled job. We use a new, lower priority queue to schedule these pruning jobs so they don't interfere with asychronous submission parsing. This commit moves the remove_builds function (and related helper functions) from common.php to DatabaseCleanupUtils. It also removes the "Cleanup database" from upgrade.php and moves this functionality to a new Artisan command instead: db:cleanup. We use this new db:cleanup command to remove shared records, such as testoutput. This allows us to remove a big chunk of custom logic from our removeBuilds funcion. While writing this commit, the following tables were already handled by db:cleanup: - buildfailuredetails - configure - configureerror - coveragefile - test2image The following tables represent potentially shared data that wasn't already handled by db:cleanup: - note - buildupdate - testoutput - updatefile - image - uploadfile The following tables were found to already have cascade-on-delete foreign keys, and thus their explicit DELETE logic was deemed safe to remove: - build2uploadfile - dynamicanalysisdefect - label2dynamicanalysis - label2buildfailure
- Loading branch information
1 parent
236cfd4
commit c0ed119
Showing
52 changed files
with
388 additions
and
835 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| <?php | ||
|
|
||
| namespace App\Console\Commands; | ||
|
|
||
| use Illuminate\Console\Command; | ||
| use Illuminate\Support\Facades\DB; | ||
|
|
||
| class CleanupDatabase extends Command | ||
| { | ||
| /** | ||
| * The name and signature of the console command. | ||
| */ | ||
| protected $signature = 'db:cleanup'; | ||
|
|
||
| /** | ||
| * The console command description. | ||
| */ | ||
| protected $description = 'Prune unused records from the CDash database'; | ||
|
|
||
| /** | ||
| * Execute the console command. | ||
| */ | ||
| public function handle(): void | ||
| { | ||
| DB::delete("DELETE FROM banner WHERE projectid != 0 AND projectid NOT IN (SELECT id FROM project)"); | ||
| self::delete_unused_rows('dailyupdate', 'projectid', 'project'); | ||
|
|
||
| self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid'); | ||
| self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid'); | ||
| self::delete_unused_rows('configureerror', 'configureid', 'configure'); | ||
| self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate'); | ||
| self::delete_unused_rows('note', 'id', 'build2note', 'noteid'); | ||
| self::delete_unused_rows('testoutput', 'id', 'build2test', 'outputid'); | ||
| self::delete_unused_rows('updatefile', 'updateid', 'buildupdate'); | ||
| self::delete_unused_rows('uploadfile', 'id', 'build2uploadfile', 'fileid'); | ||
|
|
||
| self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject'); | ||
|
|
||
| self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid'); | ||
|
|
||
| self::delete_unused_rows('test2image', 'outputid', 'testoutput'); | ||
|
|
||
| DB::delete("DELETE FROM image WHERE | ||
| id NOT IN (SELECT imageid FROM project) AND | ||
| id NOT IN (SELECT imgid FROM test2image)"); | ||
| } | ||
|
|
||
| /** Delete unused rows in batches */ | ||
| private static function delete_unused_rows(string $table, string $field, string $targettable, string $selectfield = 'id'): void | ||
| { | ||
| $start = DB::table($table)->min($field); | ||
| $max = DB::table($table)->max($field); | ||
| if (!is_numeric($start) || !is_numeric($max)) { | ||
| echo "Could not determine min and max for $field on $table\n"; | ||
| return; | ||
| } | ||
|
|
||
| $start = intval($start); | ||
| $max = intval($max); | ||
|
|
||
| $total = $max - $start; | ||
| if ($total < 1) { | ||
| return; | ||
| } | ||
| $num_done = 0; | ||
| $next_report = 10; | ||
| $done = false; | ||
| echo "Pruning unused rows from $table\n"; | ||
| while (!$done) { | ||
| $end = $start + 49999; | ||
| DB::delete(" | ||
| DELETE FROM $table | ||
| WHERE $field BETWEEN $start AND $end | ||
| AND $field NOT IN (SELECT $selectfield FROM $targettable)"); | ||
| $num_done += 50000; | ||
| if ($end >= $max) { | ||
| $done = true; | ||
| } else { | ||
| usleep(1); | ||
| $start += 50000; | ||
| // Calculate percentage of work completed so far. | ||
| $percent = round(($num_done / $total) * 100, -1); | ||
| if ($percent > $next_report) { | ||
| echo "{$percent}%\n"; | ||
| $next_report = $next_report + 10; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| <?php | ||
|
|
||
| namespace App\Jobs; | ||
|
|
||
| use Illuminate\Bus\Queueable; | ||
| use Illuminate\Contracts\Queue\ShouldQueue; | ||
| use Illuminate\Foundation\Bus\Dispatchable; | ||
| use Illuminate\Queue\InteractsWithQueue; | ||
| use Illuminate\Queue\SerializesModels; | ||
| use Illuminate\Support\Facades\Artisan; | ||
|
|
||
| /** | ||
| * Removes builds that have expired according to per-project and | ||
| * per-buildgroup settings. | ||
| */ | ||
| class PruneBuilds implements ShouldQueue | ||
| { | ||
| use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
|
||
| /** | ||
| * Execute the job. | ||
| */ | ||
| public function handle(): void | ||
| { | ||
| if (!(bool) config('cdash.autoremove_builds')) { | ||
| return; | ||
| } | ||
| Artisan::call('build:remove all'); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| <?php | ||
|
|
||
| namespace App\Jobs; | ||
|
|
||
| use Illuminate\Bus\Queueable; | ||
| use Illuminate\Contracts\Queue\ShouldQueue; | ||
| use Illuminate\Foundation\Bus\Dispatchable; | ||
| use Illuminate\Queue\InteractsWithQueue; | ||
| use Illuminate\Queue\SerializesModels; | ||
| use Illuminate\Support\Facades\Artisan; | ||
|
|
||
| /** | ||
| * Remove unreferenced database rows. | ||
| */ | ||
| class PruneDatabase implements ShouldQueue | ||
| { | ||
| use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | ||
|
|
||
| /** | ||
| * Execute the job. | ||
| */ | ||
| public function handle(): void | ||
| { | ||
| Artisan::call('db:cleanup'); | ||
| } | ||
| } |
Oops, something went wrong.