Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Do not ivalidate the cache entirely on lock file change #744

Merged
merged 3 commits into from
Jun 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 83 additions & 1 deletion .github/workflows/e2e-cache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ jobs:
- uses: actions/checkout@v3

- name: prepare sub-projects
run: __tests__/prepare-subprojects.sh
run: __tests__/prepare-yarn-subprojects.sh yarn1

# expect
# - no errors
Expand All @@ -161,3 +161,85 @@ jobs:
cache-dependency-path: |
**/*.lock
yarn.lock

yarn-subprojects-berry-local:
name: Test yarn subprojects all locally managed
strategy:
matrix:
node-version: [12, 14, 16]
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3

- name: prepare sub-projects
run: __tests__/prepare-yarn-subprojects.sh

# expect
# - no errors
# - log
# ##[info]All dependencies are managed locally by yarn3, the previous cache can be used
# ##[debug]["node-cache-Linux-yarn-401024703386272f1a950c9f014cbb1bb79a7a5b6e1fb00e8b90d06734af41ee","node-cache-Linux-yarn"]
- name: Setup Node
uses: ./
with:
node-version: ${{ matrix.node-version }}
cache: 'yarn'
cache-dependency-path: |
sub2/*.lock
sub3/*.lock

yarn-subprojects-berry-global:
name: Test yarn subprojects some locally managed
strategy:
matrix:
node-version: [12, 14, 16]
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3

- name: prepare sub-projects
run: __tests__/prepare-yarn-subprojects.sh global

# expect
# - no errors
# - log must
# ##[debug]"/home/runner/work/setup-node-test/setup-node-test/sub2" dependencies are managed by yarn 3 locally
# ##[debug]"/home/runner/work/setup-node-test/setup-node-test/sub3" dependencies are not managed by yarn 3 locally
- name: Setup Node
uses: ./
with:
node-version: ${{ matrix.node-version }}
cache: 'yarn'
cache-dependency-path: |
sub2/*.lock
sub3/*.lock

yarn-subprojects-berry-git:
name: Test yarn subprojects managed by git
strategy:
matrix:
node-version: [12, 14, 16]
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3

- name: prepare sub-projects
run: /bin/bash __tests__/prepare-yarn-subprojects.sh keepcache

# expect
# - no errors
# - log
# [debug]"/home/runner/work/setup-node-test/setup-node-test/sub2" has .yarn/cache - dependencies are kept in the repository
# [debug]"/home/runner/work/setup-node-test/setup-node-test/sub3" has .yarn/cache - dependencies are kept in the repository
# [debug]["node-cache-Linux-yarn-401024703386272f1a950c9f014cbb1bb79a7a5b6e1fb00e8b90d06734af41ee"]
- name: Setup Node
uses: ./
with:
node-version: ${{ matrix.node-version }}
cache: 'yarn'
cache-dependency-path: |
sub2/*.lock
sub3/*.lock
5 changes: 4 additions & 1 deletion __tests__/cache-utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import {
PackageManagerInfo,
isCacheFeatureAvailable,
supportedPackageManagers,
getCommandOutput
getCommandOutput,
resetProjectDirectoriesMemoized
} from '../src/cache-utils';
import fs from 'fs';
import * as cacheUtils from '../src/cache-utils';
Expand Down Expand Up @@ -103,6 +104,8 @@ describe('cache-utils', () => {
(pattern: string): Promise<Globber> =>
MockGlobber.create(['/foo', '/bar'])
);

resetProjectDirectoriesMemoized();
});

afterEach(() => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,20 @@ cat <<EOT >package.json
EOT
yarn set version 3.5.1
yarn install
if [ x$1 = 'xglobal' ];then
echo enableGlobalCache
echo 'enableGlobalCache: true' >> .yarnrc.yml
fi

echo "create yarn1 project in the root"
cd ..
cat <<EOT >package.json
if [ x$1 != 'xkeepcache' -a x$2 != 'xkeepcache' ]; then
rm -rf sub2/.yarn/cache
rm -rf sub3/.yarn/cache
fi

if [ x$1 = 'xyarn1' ];then
echo "create yarn1 project in the root"
cat <<EOT >package.json
{
"name": "subproject",
"dependencies": {
Expand All @@ -44,5 +54,6 @@ cat <<EOT >package.json
}
}
EOT
yarn set version 1.22.19
yarn install
yarn set version 1.22.19
yarn install
fi
71 changes: 69 additions & 2 deletions dist/cache-save/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -60434,7 +60434,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarnBerryManagedDependencies = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
const core = __importStar(__nccwpck_require__(2186));
const exec = __importStar(__nccwpck_require__(1514));
const cache = __importStar(__nccwpck_require__(7799));
Expand Down Expand Up @@ -60503,6 +60503,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
}
});
exports.getPackageManagerInfo = getPackageManagerInfo;
/**
* getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
* - first through `getCacheDirectories`
* - second from `repoHasYarn3ManagedCache`
*
* it contains expensive IO operation and thus should be memoized
*/
let projectDirectoriesMemoized = null;
/**
* unit test must reset memoized variables
*/
const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null);
exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized;
/**
* Expands (converts) the string input `cache-dependency-path` to list of directories that
* may be project roots
Expand All @@ -60511,6 +60524,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo;
* @return list of directories and possible
*/
const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
if (projectDirectoriesMemoized !== null) {
return projectDirectoriesMemoized;
}
const globber = yield glob.create(cacheDependencyPath);
const cacheDependenciesPaths = yield globber.glob();
const existingDirectories = cacheDependenciesPaths
Expand All @@ -60519,6 +60535,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
.filter(directory => fs_1.default.lstatSync(directory).isDirectory());
if (!existingDirectories.length)
core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`);
projectDirectoriesMemoized = existingDirectories;
return existingDirectories;
});
/**
Expand All @@ -60531,7 +60548,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
const getCacheDirectoriesFromCacheDependencyPath = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
const projectDirectories = yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath);
const cacheFoldersPaths = yield Promise.all(projectDirectories.map((projectDirectory) => __awaiter(void 0, void 0, void 0, function* () {
const cacheFolderPath = packageManagerInfo.getCacheFolderPath(projectDirectory);
const cacheFolderPath = yield packageManagerInfo.getCacheFolderPath(projectDirectory);
core.debug(`${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the directory "${projectDirectory}"`);
return cacheFolderPath;
})));
Expand Down Expand Up @@ -60565,6 +60582,56 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await
return getCacheDirectoriesForRootProject(packageManagerInfo);
});
exports.getCacheDirectories = getCacheDirectories;
/**
* A function to check if the directory is a yarn project configured to manage
* obsolete dependencies in the local cache
* @param directory - a path to the folder
* @return - true if the directory's project is yarn managed
* - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
* - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
* - if local cache is not explicitly enabled (not yarn3), return false
* - return true otherwise
*/
const projectHasYarnBerryManagedDependencies = (directory) => __awaiter(void 0, void 0, void 0, function* () {
const workDir = directory || process.env.GITHUB_WORKSPACE || '.';
core.debug(`check if "${workDir}" has locally managed yarn3 dependencies`);
// if .yarn/cache directory exists the cache is managed by version control system
const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache');
if (fs_1.default.existsSync(yarnCacheFile) &&
fs_1.default.lstatSync(yarnCacheFile).isDirectory()) {
core.debug(`"${workDir}" has .yarn/cache - dependencies are kept in the repository`);
return Promise.resolve(false);
}
// NOTE: yarn1 returns 'undefined' with return code = 0
const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir);
// only local cache is not managed by yarn
const managed = enableGlobalCache.includes('false');
if (managed) {
core.debug(`"${workDir}" dependencies are managed by yarn 3 locally`);
return true;
}
else {
core.debug(`"${workDir}" dependencies are not managed by yarn 3 locally`);
return false;
}
});
/**
* A function to report the repo contains Yarn managed projects
* @param packageManagerInfo - used to make sure current package manager is yarn
* @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
* expected to be the result of `core.getInput('cache-dependency-path')`
* @return - true if all project directories configured to be Yarn managed
*/
const repoHasYarnBerryManagedDependencies = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
if (packageManagerInfo.name !== 'yarn')
return false;
const yarnDirs = cacheDependencyPath
? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
: [''];
const isManagedList = yield Promise.all(yarnDirs.map(projectHasYarnBerryManagedDependencies));
return isManagedList.every(Boolean);
});
exports.repoHasYarnBerryManagedDependencies = repoHasYarnBerryManagedDependencies;
function isGhes() {
const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com');
return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';
Expand Down
84 changes: 80 additions & 4 deletions dist/setup/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -71153,10 +71153,19 @@ const restoreCache = (packageManager, cacheDependencyPath) => __awaiter(void 0,
if (!fileHash) {
throw new Error('Some specified paths were not resolved, unable to cache dependencies.');
}
const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`;
const keyPrefix = `node-cache-${platform}-${packageManager}`;
const primaryKey = `${keyPrefix}-${fileHash}`;
core.debug(`primary key is ${primaryKey}`);
core.saveState(constants_1.State.CachePrimaryKey, primaryKey);
const cacheKey = yield cache.restoreCache(cachePaths, primaryKey);
const isManagedByYarnBerry = yield cache_utils_1.repoHasYarnBerryManagedDependencies(packageManagerInfo, cacheDependencyPath);
let cacheKey;
if (isManagedByYarnBerry) {
core.info('All dependencies are managed locally by yarn3, the previous cache can be used');
cacheKey = yield cache.restoreCache(cachePaths, primaryKey, [keyPrefix]);
}
else {
cacheKey = yield cache.restoreCache(cachePaths, primaryKey);
}
core.setOutput('cache-hit', Boolean(cacheKey));
if (!cacheKey) {
core.info(`${packageManager} cache is not found`);
Expand Down Expand Up @@ -71217,7 +71226,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarnBerryManagedDependencies = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
const core = __importStar(__nccwpck_require__(2186));
const exec = __importStar(__nccwpck_require__(1514));
const cache = __importStar(__nccwpck_require__(7799));
Expand Down Expand Up @@ -71286,6 +71295,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
}
});
exports.getPackageManagerInfo = getPackageManagerInfo;
/**
* getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
* - first through `getCacheDirectories`
* - second from `repoHasYarn3ManagedCache`
*
* it contains expensive IO operation and thus should be memoized
*/
let projectDirectoriesMemoized = null;
/**
* unit test must reset memoized variables
*/
const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null);
exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized;
/**
* Expands (converts) the string input `cache-dependency-path` to list of directories that
* may be project roots
Expand All @@ -71294,6 +71316,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo;
* @return list of directories and possible
*/
const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
if (projectDirectoriesMemoized !== null) {
return projectDirectoriesMemoized;
}
const globber = yield glob.create(cacheDependencyPath);
const cacheDependenciesPaths = yield globber.glob();
const existingDirectories = cacheDependenciesPaths
Expand All @@ -71302,6 +71327,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
.filter(directory => fs_1.default.lstatSync(directory).isDirectory());
if (!existingDirectories.length)
core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`);
projectDirectoriesMemoized = existingDirectories;
return existingDirectories;
});
/**
Expand All @@ -71314,7 +71340,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
const getCacheDirectoriesFromCacheDependencyPath = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
const projectDirectories = yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath);
const cacheFoldersPaths = yield Promise.all(projectDirectories.map((projectDirectory) => __awaiter(void 0, void 0, void 0, function* () {
const cacheFolderPath = packageManagerInfo.getCacheFolderPath(projectDirectory);
const cacheFolderPath = yield packageManagerInfo.getCacheFolderPath(projectDirectory);
core.debug(`${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the directory "${projectDirectory}"`);
return cacheFolderPath;
})));
Expand Down Expand Up @@ -71348,6 +71374,56 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await
return getCacheDirectoriesForRootProject(packageManagerInfo);
});
exports.getCacheDirectories = getCacheDirectories;
/**
* A function to check if the directory is a yarn project configured to manage
* obsolete dependencies in the local cache
* @param directory - a path to the folder
* @return - true if the directory's project is yarn managed
* - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
* - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
* - if local cache is not explicitly enabled (not yarn3), return false
* - return true otherwise
*/
const projectHasYarnBerryManagedDependencies = (directory) => __awaiter(void 0, void 0, void 0, function* () {
const workDir = directory || process.env.GITHUB_WORKSPACE || '.';
core.debug(`check if "${workDir}" has locally managed yarn3 dependencies`);
// if .yarn/cache directory exists the cache is managed by version control system
const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache');
if (fs_1.default.existsSync(yarnCacheFile) &&
fs_1.default.lstatSync(yarnCacheFile).isDirectory()) {
core.debug(`"${workDir}" has .yarn/cache - dependencies are kept in the repository`);
return Promise.resolve(false);
}
// NOTE: yarn1 returns 'undefined' with return code = 0
const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir);
// only local cache is not managed by yarn
const managed = enableGlobalCache.includes('false');
if (managed) {
core.debug(`"${workDir}" dependencies are managed by yarn 3 locally`);
return true;
}
else {
core.debug(`"${workDir}" dependencies are not managed by yarn 3 locally`);
return false;
}
});
/**
* A function to report the repo contains Yarn managed projects
* @param packageManagerInfo - used to make sure current package manager is yarn
* @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
* expected to be the result of `core.getInput('cache-dependency-path')`
* @return - true if all project directories configured to be Yarn managed
*/
const repoHasYarnBerryManagedDependencies = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
if (packageManagerInfo.name !== 'yarn')
return false;
const yarnDirs = cacheDependencyPath
? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
: [''];
const isManagedList = yield Promise.all(yarnDirs.map(projectHasYarnBerryManagedDependencies));
return isManagedList.every(Boolean);
});
exports.repoHasYarnBerryManagedDependencies = repoHasYarnBerryManagedDependencies;
function isGhes() {
const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com');
return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';
Expand Down
Loading