Skip to content

Commit

Permalink
Vendors in olsen232/arbiter assume-role-with-web-identity
Browse files Browse the repository at this point in the history 
Arbiter commit: 97c1d17522
Description:
    Implements AssumeRoleWithWebIdentity

    Steps taken are the same as in GDAL:

    Before trying IMDS, but after all other types of auth,
    - see if AWS_ROLE_ARN and AWS_WEB_IDENTITY_TOKEN_FILE are set
    - if so, load the AWS_WEB_IDENTITY_TOKEN_FILE
    - use it to construct an STS AssumeRoleWithWebIdentity URL
    - use that URL in a similar manner to the other token-issuing URLs
    - except that the response is XML, not JSON
  • Loading branch information
@olsen232
olsen232 committed Sep 24, 2024
1 parent 22b115b commit 313a0c7
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 24 deletions.
134 changes: 114 additions & 20 deletions vendor/arbiter/arbiter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,16 +353,13 @@ std::shared_ptr<Driver> Arbiter::getDriver(const std::string path) const
{
const auto type(getProtocol(path));

{
std::lock_guard<std::mutex> lock(m_mutex);
auto it = m_drivers.find(type);
if (it != m_drivers.end()) return it->second;
}
std::lock_guard<std::mutex> lock(m_mutex);
auto it = m_drivers.find(type);
if (it != m_drivers.end()) return it->second;

const json config = getConfig(m_config);
if (auto driver = Driver::create(*m_pool, type, config.dump()))
{
std::lock_guard<std::mutex> lock(m_mutex);
m_drivers[type] = driver;
return driver;
}
Expand Down Expand Up @@ -1867,9 +1864,55 @@ std::unique_ptr<S3::Auth> S3::Auth::create(
drivers::Http httpDriver(pool);

// Nothing found in the environment or on the filesystem. However we may
// be running in an EC2 instance with an instance profile set up.
// be running in an EC2 instance with a service account or an instance profile set up.
try
{
// Try to "assume role with web identity" - see https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
try
{
const auto roleArn = env("AWS_ROLE_ARN");
const auto webIdentityTokenFile = env("AWS_WEB_IDENTITY_TOKEN_FILE");
std::unique_ptr<std::string> webIdentityToken;

if (roleArn && webIdentityTokenFile && (webIdentityToken = fsDriver.tryGet(*webIdentityTokenFile)))
{
// Decide on the STS root URL, defaults to https://sts.<AWS_REGION>.amazonaws.com
std::string stsRootUrl;
if (env("AWS_STS_ROOT_URL"))
{
stsRootUrl = *env("AWS_STS_ROOT_URL");
}
else
{
bool useRegionalEndpoint = env("AWS_STS_REGIONAL_ENDPOINTS")
? (*env("AWS_STS_REGIONAL_ENDPOINTS") == "regional")
: true;
if (useRegionalEndpoint)
{
stsRootUrl = "https://sts." + S3::Config::extractRegion(s, profile) + ".amazonaws.com";
}
else
{
stsRootUrl = "https://sts.amazonaws.com";
}
}

const std::string stsAssumeRoleWithWebIdentityUrl = stsRootUrl
+ "/?Action=AssumeRoleWithWebIdentity&RoleSessionName=pdal&Version=2011-06-15"
+ "&RoleArn=" + *roleArn
+ "&WebIdentityToken=" + *webIdentityToken;

const auto res = httpDriver.internalGet(stsAssumeRoleWithWebIdentityUrl);
if (!res.ok())
{
throw ArbiterError("Failed to assume role with web identity");
}

return makeUnique<Auth>(stsAssumeRoleWithWebIdentityUrl, ReauthMethod::ASSUME_ROLE_WITH_WEB_IDENTITY);
}
}
catch (...) { }

std::string token;

try
Expand Down Expand Up @@ -1911,8 +1954,8 @@ std::unique_ptr<S3::Auth> S3::Auth::create(

if (!iamRole.empty())
{
const bool imdsv2 = !token.empty();
return makeUnique<Auth>(ec2CredBase + "/" + iamRole, imdsv2);
const ReauthMethod reauthMethod = !token.empty() ? ReauthMethod::IMDS_V2 : ReauthMethod::IMDS_V1;
return makeUnique<Auth>(ec2CredBase + "/" + iamRole, reauthMethod);
}
}
catch (...) { }
Expand All @@ -1921,7 +1964,7 @@ std::unique_ptr<S3::Auth> S3::Auth::create(
// different IP.
if (const auto relUri = env("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"))
{
return makeUnique<Auth>(fargateCredIp + "/" + *relUri);
return makeUnique<Auth>(fargateCredIp + "/" + *relUri, ReauthMethod::IMDS_V2);
}
#endif

Expand Down Expand Up @@ -2090,7 +2133,7 @@ S3::AuthFields S3::Auth::fields() const

std::string token;

if (m_imdsv2)
if (m_reauthMethod == ReauthMethod::IMDS_V2)
{
try
{
Expand All @@ -2116,16 +2159,67 @@ S3::AuthFields S3::Auth::fields() const
http::Headers headers;
if (!token.empty()) headers["X-aws-ec2-metadata-token"] = token;

const json creds = json::parse(
httpDriver.get(*m_credUrl, headers));
const auto res = httpDriver.internalGet(*m_credUrl, headers);
if (!res.ok())
{
throw ArbiterError("Failed to get token");
}
std::vector<char> data(res.data());
data.push_back('\0');

if (m_reauthMethod == ReauthMethod::ASSUME_ROLE_WITH_WEB_IDENTITY)
{
// Parse XML response.
Xml::xml_document<> xml;
try
{
xml.parse<0>(data.data());
}
catch (Xml::parse_error&)
{
throw ArbiterError("Could not parse S3 response.");
}
bool parsed = false;
if (XmlNode* topNode = xml.first_node("AssumeRoleWithWebIdentityResponse"))
{
if (XmlNode* resultNode = topNode->first_node("AssumeRoleWithWebIdentityResult"))
{
if (XmlNode* credsNode = resultNode->first_node("Credentials"))
{
XmlNode* accessNode = credsNode->first_node("AccessKeyId");
XmlNode* hiddenNode = credsNode->first_node("SecretAccessKey");
XmlNode* tokenNode = credsNode->first_node("SessionToken");
XmlNode* expirationNode = credsNode->first_node("Expiration");
if (accessNode && hiddenNode && tokenNode && expirationNode)
{
m_access = accessNode->value();
m_hidden = hiddenNode->value();
m_token = tokenNode->value();
m_expiration.reset(new Time(expirationNode->value(), arbiter::Time::iso8601));
parsed = true;
}
}
}
}
if (!parsed)
{
throw ArbiterError("Could not parse S3 response.");
}

m_access = creds.at("AccessKeyId").get<std::string>();
m_hidden = creds.at("SecretAccessKey").get<std::string>();
m_token = creds.at("Token").get<std::string>();
m_expiration.reset(
new Time(
creds.at("Expiration").get<std::string>(),
arbiter::Time::iso8601));
}
else
{
// Parse JSON response.
const json creds = json::parse(res.data());

m_access = creds.at("AccessKeyId").get<std::string>();
m_hidden = creds.at("SecretAccessKey").get<std::string>();
m_token = creds.at("Token").get<std::string>();
m_expiration.reset(
new Time(
creds.at("Expiration").get<std::string>(),
arbiter::Time::iso8601));
}

if (*m_expiration - now < reauthSeconds)
{
Expand Down
17 changes: 13 additions & 4 deletions vendor/arbiter/arbiter.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/// Arbiter amalgamated header (https://github.com/connormanning/arbiter).
/// It is intended to be used with #include "arbiter.hpp"

// Git SHA: 4d61535996946414317c6617724c20a2bc4d9bbf
// Git SHA: 5c3f36e86e7a74aadb8a98edb14ad207158aa785

// //////////////////////////////////////////////////////////////////////
// Beginning of content of file: LICENSE
Expand Down Expand Up @@ -4225,6 +4225,12 @@ namespace arbiter
namespace drivers
{

enum class ReauthMethod {
ASSUME_ROLE_WITH_WEB_IDENTITY,
IMDS_V1,
IMDS_V2,
};

/** @brief Amazon %S3 driver. */
class S3 : public Http
{
Expand All @@ -4245,6 +4251,7 @@ class S3 : public Http
* - JSON configuration
* - Well-known files or their environment overrides, like
* `~/.aws/credentials` or the file at AWS_CREDENTIAL_FILE.
* - STS assume role with web identity.
* - EC2 instance profile.
*/
static std::unique_ptr<S3> create(
Expand Down Expand Up @@ -4316,9 +4323,9 @@ class S3::Auth
, m_token(token)
{ }

Auth(std::string credUrl, bool imdsv2 = true)
Auth(std::string credUrl, ReauthMethod reauthMethod)
: m_credUrl(internal::makeUnique<std::string>(credUrl))
, m_imdsv2(imdsv2)
, m_reauthMethod(reauthMethod)
{ }

static std::unique_ptr<Auth> create(std::string profile, std::string s);
Expand All @@ -4331,7 +4338,7 @@ class S3::Auth
mutable std::string m_token;

std::unique_ptr<std::string> m_credUrl;
bool m_imdsv2 = true;
ReauthMethod m_reauthMethod;
mutable std::unique_ptr<Time> m_expiration;
mutable std::mutex m_mutex;
};
Expand All @@ -4354,6 +4361,8 @@ class S3::Config
const std::string m_baseUrl;
http::Headers m_baseHeaders;
bool m_precheck;

friend class S3;
};


Expand Down

0 comments on commit 313a0c7

Please sign in to comment.