2025
2024-11-25 AWScli - Assume role and export it as temporary env
export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
$(aws sts assume-role \
--role-arn ${AWS_ROLE_ARN} \
--role-session-name MySessionName \
--query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
--output text))
2024-10-21 How to query AWS WAF log on S3 + Firehose
ref: https://docs.aws.amazon.com/athena/latest/ug/create-waf-table-partition-projection.html
- Create Table
CREATE EXTERNAL TABLE `waf_logs_rammus`(
`timestamp` bigint,
`formatversion` int,
`webaclid` string,
`terminatingruleid` string,
`terminatingruletype` string,
`action` string,
`terminatingrulematchdetails` array <
struct <
conditiontype: string,
sensitivitylevel: string,
location: string,
matcheddata: array < string >
>
>,
`httpsourcename` string,
`httpsourceid` string,
`rulegrouplist` array <
struct <
rulegroupid: string,
terminatingrule: struct <
ruleid: string,
action: string,
rulematchdetails: array <
struct <
conditiontype: string,
sensitivitylevel: string,
location: string,
matcheddata: array < string >
>
>
>,
nonterminatingmatchingrules: array <
struct <
ruleid: string,
action: string,
overriddenaction: string,
rulematchdetails: array <
struct <
conditiontype: string,
sensitivitylevel: string,
location: string,
matcheddata: array < string >
>
>,
challengeresponse: struct <
responsecode: string,
solvetimestamp: string
>,
captcharesponse: struct <
responsecode: string,
solvetimestamp: string
>
>
>,
excludedrules: string
>
>,
`ratebasedrulelist` array <
struct <
ratebasedruleid: string,
limitkey: string,
maxrateallowed: int
>
>,
`nonterminatingmatchingrules` array <
struct <
ruleid: string,
action: string,
rulematchdetails: array <
struct <
conditiontype: string,
sensitivitylevel: string,
location: string,
matcheddata: array < string >
>
>,
challengeresponse: struct <
responsecode: string,
solvetimestamp: string
>,
captcharesponse: struct <
responsecode: string,
solvetimestamp: string
>
>
>,
`requestheadersinserted` array <
struct <
name: string,
value: string
>
>,
`responsecodesent` string,
`httprequest` struct <
clientip: string,
country: string,
headers: array <
struct <
name: string,
value: string
>
>,
uri: string,
args: string,
httpversion: string,
httpmethod: string,
requestid: string
>,
`labels` array <
struct <
name: string
>
>,
`captcharesponse` struct <
responsecode: string,
solvetimestamp: string,
failureReason: string
>,
`challengeresponse` struct <
responsecode: string,
solvetimestamp: string,
failureReason: string
>,
`ja3Fingerprint` string,
`oversizefields` string,
`requestbodysize` int,
`requestbodysizeinspectedbywaf` int
)
PARTITIONED BY (
`year` string,
`month` string,
`day` string,
`hour` string)
ROW FORMAT SERDE
'org.openx.data.jsonserde.JsonSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
's3://eslite-waf-log-lms-api/'
TBLPROPERTIES(
'projection.enabled' = 'true',
'projection.year.type' = 'enum',
'projection.year.values' = '2024',
'projection.month.type' = 'enum',
'projection.month.values' = '10',
'projection.day.type' = 'enum',
'projection.day.values' = '21',
'projection.hour.type' = 'enum',
'projection.hour.values' = '00',
'storage.location.template' = 's3://waf-log-rammus/${year}/${month}/${day}/${hour}/')
- Test query
SELECT *
FROM waf_logs_rammus
WHERE year = '2024'
AND month = '10'
AND day = '21'
AND hour = '00'
LIMIT 100;
- Count 2024/10
SELECT
httprequest.clientip, terminatingruleid,
COUNT(*) AS request_count
FROM waf_logs_rammus
WHERE year = '2024'
AND month = '10'
GROUP BY httprequest.clientip,terminatingruleid
ORDER BY request_count DESC;
- Projection 2024 all logs
ALTER TABLE waf_logs_rammus
SET TBLPROPERTIES (
'projection.enabled' = 'true',
'projection.year.type' = 'enum',
'projection.year.values' = '2024',
'projection.month.type' = 'integer',
'projection.month.range' = '1,12',
'projection.day.type' = 'integer',
'projection.day.range' = '1,31',
'projection.hour.type' = 'integer',
'projection.hour.range' = '0,23',
'storage.location.template' = 's3://-waf-log-rammus/${year}/${month}/${day}/${hour}/'
);
2024-10-18 podman setup
brew install podman
podman machine init
podman run -dt -p 8080:80/tcp docker.io/library/httpd
curl localhost:8080
podman ps
podman machine stop
podman machine ls
2024-10-16 Azure Application Gateway vs Load Balancer
2024-10-07 透過 EKS 連線到 AWS Redis (Elastic Cache) 除錯
aws eks update-kubeconfig --region ap-northeast-1 --name rammusxu-cluster --profile rammusxu
kubectl run redis-cli --image=redis --restart=Never --rm -it -- redis-cli
root@redis-cli:/data# redis-cli -h xxx.apne1.cache.amazonaws.com --bigkeys -i 0.1
# Scanning the entire keyspace to find biggest keys as well as
# average sizes per key type. You can use -i 0.1 to sleep 0.1 sec
# per 100 SCAN commands (not usually needed).
5.89% ||||--------------------------------------------------------
11.81% |||||||-----------------------------------------------------
17.68% |||||||||||-------------------------------------------------
23.58% ||||||||||||||----------------------------------------------
29.44% ||||||||||||||||||------------------------------------------
35.33% |||||||||||||||||||||---------------------------------------
41.23% |||||||||||||||||||||||||-----------------------------------
47.08% ||||||||||||||||||||||||||||--------------------------------
50.15% ||||||||||||||||||||||||||||||------------------------------
55.96% ||||||||||||||||||||||||||||||||||--------------------------
61.84% |||||||||||||||||||||||||||||||||||||-----------------------
67.76% |||||||||||||||||||||||||||||||||||||||||-------------------
73.61% ||||||||||||||||||||||||||||||||||||||||||||----------------
79.16% |||||||||||||||||||||||||||||||||||||||||||||||-------------
82.50% |||||||||||||||||||||||||||||||||||||||||||||||||-----------
88.41% |||||||||||||||||||||||||||||||||||||||||||||||||||||-------
94.31% |||||||||||||||||||||||||||||||||||||||||||||||||||||||||---
100.00% ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
Keys sampled: 34916
-------- summary -------
Total key length in bytes is 2951250 (avg len 84.52)
Biggest string found "_promotion_7704278_1" has 1531728 bytes
0 lists with 0 items (00.00% of keys, avg size 0.00)
0 hashs with 0 fields (00.00% of keys, avg size 0.00)
0 streams with 0 entries (00.00% of keys, avg size 0.00)
34916 strings with 40322416924 bytes (100.00% of keys, avg size 1154840.67)
0 sets with 0 members (00.00% of keys, avg size 0.00)
0 zsets with 0 members (00.00% of keys, avg size 0.00)
--bigkeys會 Scan 所有 key- 必須結合
-i 0.1sleep，避免生產環境被阻塞