Skip to content

Commit

Permalink
Move remove_builds (and related functions) to DatabaseUtils (#2575)
Browse files Browse the repository at this point in the history 
Continue to reduce the amount of legacy code that lives in common.php.
Functionally, this should be more-or-less a no-op.
  • Loading branch information
@zackgalbreath
zackgalbreath authored Nov 20, 2024
1 parent a0061f0 commit 89eb9a2
Show file tree
Hide file tree
Showing 39 changed files with 471 additions and 405 deletions.
5 changes: 3 additions & 2 deletions app/Http/Controllers/AdminController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
namespace App\Http\Controllers;

use App\Models\User;
use App\Utils\DatabaseCleanupUtils;
use CDash\Model\Project;
use Illuminate\Http\RedirectResponse;
use Illuminate\Support\Facades\DB;
Expand Down Expand Up @@ -96,7 +97,7 @@ public function removeBuilds(): View|RedirectResponse
$builds[] = (int) $build_array->id;
}

remove_build_chunked($builds);
DatabaseCleanupUtils::removeBuildChunked($builds);
$alert = 'Removed ' . count($builds) . ' builds.';
}

Expand Down Expand Up @@ -446,7 +447,7 @@ public function upgrade()
starttime<'1975-12-31 23:59:59' OR starttime>'$forwarddate'");
while ($builds_array = pdo_fetch_array($builds)) {
$buildid = $builds_array['id'];
remove_build($buildid);
DatabaseCleanupUtils::removeBuild($buildid);
}
}

Expand Down
3 changes: 2 additions & 1 deletion app/Http/Controllers/BuildController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
use App\Models\Comment;
use App\Models\User;
use App\Models\Build as EloquentBuild;
use App\Utils\DatabaseCleanupUtils;
use App\Utils\PageTimer;
use App\Utils\RepositoryUtils;
use App\Utils\TestingDay;
Expand Down Expand Up @@ -1470,7 +1471,7 @@ private function restApiPost(): JsonResponse
private function restApiDelete(): JsonResponse
{
Log::info("Build #{$this->build->Id} removed manually.");
remove_build($this->build->Id);
DatabaseCleanupUtils::removeBuild($this->build->Id);
return response()->json();
}
}
312 changes: 310 additions & 2 deletions app/Utils/DatabaseCleanupUtils.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@

use App\Models\Build;
use App\Models\BuildGroup;
use CDash\Database;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use InvalidArgumentException;

class DatabaseCleanupUtils
{
Expand Down Expand Up @@ -54,7 +56,7 @@ public static function removeBuildsGroupwise(int $projectid, int $maxbuilds, boo
$s = 'removing old buildids for projectid: ' . $projectid;
Log::info($s);
echo ' -- ' . $s . "\n";
remove_build_chunked($buildids);
self::removeBuildChunked($buildids);
}

/** Remove the first builds that are at the beginning of the queue */
Expand Down Expand Up @@ -89,6 +91,312 @@ public static function removeFirstBuilds(int $projectid, int $days, int $maxbuil
if ($echo) {
echo ' -- ' . $s . "\n"; // for "interactive" command line feedback
}
remove_build_chunked($buildids);
self::removeBuildChunked($buildids);
}

/**
* Remove all related inserts for a given build or any build in an array of builds
* @param array<int>|int $buildid
* @throws \InvalidArgumentException
*/
public static function removeBuild($buildid) : void
{
// TODO: (williamjallen) much of this work could be done on the DB side automatically by setting up
// proper foreign-key relationships between between entities, and using the DB's cascade functionality.
// For complex cascades, custom SQL functions can be written.

if (!is_array($buildid)) {
$buildid = [$buildid];
}

$buildids = [];
foreach ($buildid as $b) {
if (!is_numeric($b)) {
throw new InvalidArgumentException('Invalid Build ID');
}
$buildids[] = intval($b);
}

$db = Database::getInstance();
$buildid_prepare_array = $db->createPreparedArray(count($buildids));

// Remove the buildfailureargument
$buildfailureids = [];
$buildfailure = DB::select("SELECT id FROM buildfailure WHERE buildid IN $buildid_prepare_array", $buildids);
foreach ($buildfailure as $buildfailure_array) {
$buildfailureids[] = intval($buildfailure_array->id);
}
if (count($buildfailureids) > 0) {
$buildfailure_prepare_array = $db->createPreparedArray(count($buildfailureids));
DB::delete("DELETE FROM buildfailure2argument WHERE buildfailureid IN $buildfailure_prepare_array", $buildfailureids);
DB::delete("DELETE FROM label2buildfailure WHERE buildfailureid IN $buildfailure_prepare_array", $buildfailureids);
}

// Delete buildfailuredetails that are only used by builds that are being
// deleted.
DB::delete("
DELETE FROM buildfailuredetails WHERE id IN (
SELECT a.detailsid
FROM buildfailure AS a
LEFT JOIN buildfailure AS b ON (
a.detailsid=b.detailsid
AND b.buildid NOT IN $buildid_prepare_array
)
WHERE a.buildid IN $buildid_prepare_array
GROUP BY a.detailsid
HAVING count(b.detailsid)=0
)
", array_merge($buildids, $buildids));

// Delete the configure if not shared.
$build2configure = DB::select("
SELECT a.configureid
FROM build2configure AS a
LEFT JOIN build2configure AS b ON (
a.configureid=b.configureid
AND b.buildid NOT IN $buildid_prepare_array
)
WHERE a.buildid IN $buildid_prepare_array
GROUP BY a.configureid
HAVING count(b.configureid)=0
", array_merge($buildids, $buildids));

$configureids = [];
foreach ($build2configure as $build2configure_array) {
// It is safe to delete this configure because it is only used
// by builds that are being deleted.
$configureids[] = intval($build2configure_array->configureid);
}
if (count($configureids) > 0) {
$configureids_prepare_array = $db->createPreparedArray(count($configureids));
DB::delete("DELETE FROM configure WHERE id IN $configureids_prepare_array", $configureids);
}

// coverage files are kept unless they are shared
DB::delete("
DELETE FROM coveragefile
WHERE id IN (
SELECT f1.id
FROM (
SELECT a.fileid AS id, COUNT(DISTINCT a.buildid) AS c
FROM coverage a
WHERE a.buildid IN $buildid_prepare_array
GROUP BY a.fileid
) AS f1
INNER JOIN (
SELECT b.fileid AS id, COUNT(DISTINCT b.buildid) AS c
FROM coverage b
INNER JOIN (
SELECT fileid
FROM coverage
WHERE buildid IN $buildid_prepare_array
) AS d ON b.fileid = d.fileid
GROUP BY b.fileid
) AS f2 ON (f1.id = f2.id)
WHERE f1.c = f2.c
)
", array_merge($buildids, $buildids));

// dynamicanalysisdefect
$dynamicanalysis = DB::select("
SELECT id
FROM dynamicanalysis
WHERE buildid IN $buildid_prepare_array
", $buildids);

$dynids = [];
foreach ($dynamicanalysis as $dynamicanalysis_array) {
$dynids[] = intval($dynamicanalysis_array->id);
}

if (count($dynids) > 0) {
$dynids_prepare_array = $db->createPreparedArray(count($dynids));
DB::delete("DELETE FROM dynamicanalysisdefect WHERE dynamicanalysisid IN $dynids_prepare_array", $dynids);
DB::delete("DELETE FROM label2dynamicanalysis WHERE dynamicanalysisid IN $dynids_prepare_array", $dynids);
}

// Delete the note if not shared
DB::delete("
DELETE FROM note WHERE id IN (
SELECT f1.id
FROM (
SELECT a.noteid AS id, COUNT(DISTINCT a.buildid) AS c
FROM build2note a
WHERE a.buildid IN $buildid_prepare_array
GROUP BY a.noteid
) AS f1
INNER JOIN (
SELECT b.noteid AS id, COUNT(DISTINCT b.buildid) AS c
FROM build2note b
INNER JOIN (
SELECT noteid
FROM build2note
WHERE buildid IN $buildid_prepare_array
) AS d ON b.noteid = d.noteid
GROUP BY b.noteid
) AS f2 ON (f1.id = f2.id)
WHERE f1.c = f2.c
)
", array_merge($buildids, $buildids));

// Delete the update if not shared
$build2update = DB::select("
SELECT a.updateid
FROM build2update AS a
LEFT JOIN build2update AS b ON (
a.updateid=b.updateid
AND b.buildid NOT IN $buildid_prepare_array
)
WHERE a.buildid IN $buildid_prepare_array
GROUP BY a.updateid
HAVING count(b.updateid)=0
", array_merge($buildids, $buildids));

$updateids = [];
foreach ($build2update as $build2update_array) {
// Update is not shared we delete
$updateids[] = intval($build2update_array->updateid);
}

if (count($updateids) > 0) {
$updateids_prepare_array = $db->createPreparedArray(count($updateids));
DB::delete("DELETE FROM buildupdate WHERE id IN $updateids_prepare_array", $updateids);
DB::delete("DELETE FROM updatefile WHERE updateid IN $updateids_prepare_array", $updateids);
}

// Delete tests and testoutputs that are not shared.
// First find all the tests and testoutputs from builds that are about to be deleted.
$b2t_result = DB::select("
SELECT DISTINCT outputid
FROM build2test
WHERE buildid IN $buildid_prepare_array
", $buildids);

$all_outputids = [];
foreach ($b2t_result as $b2t_row) {
$all_outputids[] = intval($b2t_row->outputid);
}

// Delete un-shared testoutput rows.
if (!empty($all_outputids)) {
// Next identify tests from this list that should be preserved
// because they are shared with builds that are not about to be deleted.
$all_outputids_prepare_array = $db->createPreparedArray(count($all_outputids));
$save_test_result = DB::select("
SELECT DISTINCT outputid
FROM build2test
WHERE
outputid IN $all_outputids_prepare_array
AND buildid NOT IN $buildid_prepare_array
", array_merge($all_outputids, $buildids));
$testoutputs_to_save = [];
foreach ($save_test_result as $save_test_row) {
$testoutputs_to_save[] = intval($save_test_row->outputid);
}

// Use array_diff to get the list of tests that should be deleted.
$testoutputs_to_delete = array_diff($all_outputids, $testoutputs_to_save);
if (!empty($testoutputs_to_delete)) {
self::deleteRowsChunked('DELETE FROM testoutput WHERE id IN ', $testoutputs_to_delete);

$testoutputs_to_delete_prepare_array = $db->createPreparedArray(count($testoutputs_to_delete));
// Check if the images for the test are not shared
$test2image = DB::select("
SELECT a.imgid
FROM test2image AS a
LEFT JOIN test2image AS b ON (
a.imgid=b.imgid
AND b.outputid NOT IN $testoutputs_to_delete_prepare_array
)
WHERE a.outputid IN $testoutputs_to_delete_prepare_array
GROUP BY a.imgid
HAVING count(b.imgid)=0
", array_merge($testoutputs_to_delete, $testoutputs_to_delete));

$imgids = [];
foreach ($test2image as $test2image_array) {
$imgids[] = intval($test2image_array->imgid);
}

if (count($imgids) > 0) {
$imgids_prepare_array = $db->createPreparedArray(count($imgids));
DB::delete("DELETE FROM image WHERE id IN $imgids_prepare_array", $imgids);
}
self::deleteRowsChunked('DELETE FROM test2image WHERE outputid IN ', $testoutputs_to_delete);
}
}

// Delete the uploaded files if not shared
$build2uploadfiles = DB::select("
SELECT a.fileid
FROM build2uploadfile AS a
LEFT JOIN build2uploadfile AS b ON (
a.fileid=b.fileid
AND b.buildid NOT IN $buildid_prepare_array
)
WHERE a.buildid IN $buildid_prepare_array
GROUP BY a.fileid
HAVING count(b.fileid)=0
", array_merge($buildids, $buildids));

$fileids = [];
foreach ($build2uploadfiles as $build2uploadfile_array) {
$fileid = intval($build2uploadfile_array->fileid);
$fileids[] = $fileid;
unlink_uploaded_file($fileid);
}

if (count($fileids) > 0) {
$fileids_prepare_array = $db->createPreparedArray(count($fileids));
DB::delete("DELETE FROM uploadfile WHERE id IN $fileids_prepare_array", $fileids);
DB::delete("DELETE FROM build2uploadfile WHERE fileid IN $fileids_prepare_array", $fileids);
}

// Remove any children of these builds.
// In order to avoid making the list of builds to delete too large
// we delete them in batches (one batch per parent).
foreach ($buildids as $parentid) {
$child_result = DB::select('SELECT id FROM build WHERE parentid=?', [intval($parentid)]);

$childids = [];
foreach ($child_result as $child_array) {
$childids[] = intval($child_array->id);
}
if (!empty($childids)) {
self::removeBuild($childids);
}
}

// Only delete the buildid at the end so that no other build can get it in the meantime
DB::delete("DELETE FROM build WHERE id IN $buildid_prepare_array", $buildids);

add_last_sql_error('remove_build');
}

/**
* Call removeBuild() in batches of 100.
* @param array<int>|int $buildids
*/
public static function removeBuildChunked($buildids): void
{
if (!is_array($buildids)) {
self::removeBuild($buildid);
}
foreach (array_chunk($buildids, 100) as $chunk) {
self::removeBuild($chunk);
}
}

/**
* Chunk up DELETE queries into batches of 100.
*/
private static function deleteRowsChunked(string $query, array $ids): void
{
foreach (array_chunk($ids, 100) as $chunk) {
$chunk_prepared_array = Database::getInstance()->createPreparedArray(count($chunk));
DB::delete("$query $chunk_prepared_array", $chunk);
// Sleep for a microsecond to give other processes a chance.
usleep(1);
}
}
}
2 changes: 1 addition & 1 deletion app/Utils/SubmissionUtils.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public static function add_build(Build $build)
if ($buildid > 0 && !$build->Append) {
$build->Id = $buildid;
if ($build->GetDone()) {
remove_build($buildid);
DatabaseCleanupUtils::removeBuild($buildid);
$build->Id = null;
}
}
Expand Down
Loading

0 comments on commit 89eb9a2

Please sign in to comment.