Parse two items from text file
I have a long file containing AWS AMI names / Image IDs. I would like to use python to parse both the "Name" and "ImageId". The problem is that they're not paired together, as shown in the text below.
In the example below, the Image ID is ami-00455a561ee2a2221, while the Name is temp-rp030-3.
{
"VirtualizationType": "hvm",
"Description": "",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-00455a561ee2a2221",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-097a377c34d632344",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 1500,
"Encrypted": true
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
}
],
"Architecture": "x86_64",
"ImageLocation": "051929652521/temp-rp030-3",
"RootDeviceType": "ebs",
"OwnerId": "051929652521",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2022-01-15T22:36:32.000Z",
"Public": false,
"ImageType": "machine",
"Name": "temp-rp030-3"
},
I have tried something like the following, where I search for Image ID, then print the next 18 lines into a list. However, I'm finding that not every occurence of Name is 18 lines after ImageId.
import re
from itertools import islice
iter_list = []
with open("/Users/Desktop/describe_images.txt", 'r') as f:
for line in f:
if re.search(r"\"ImageId\"", line):
iter_list.append(''.join(islice(f, 18)))
Here is a larger example contianing multiple Names and ImageID
{
"VirtualizationType": "hvm",
"Description": "",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-00455a561ee2a2221",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-097a377c34d632344",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 1500,
"Encrypted": true
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
}
],
"Architecture": "x86_64",
"ImageLocation": "051929652521/temp-rp030-3",
"RootDeviceType": "ebs",
"OwnerId": "051929652521",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2022-01-15T22:36:32.000Z",
"Public": false,
"ImageType": "machine",
"Name": "temp-rp030-3"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-00455bda4a1b36dd7",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0dac397dbf6de2dc8",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
"ImageLocation": "979382823631/bitnami-wordpress-5.7.1-7-r06-linux-debian-10-x86_64-hvm-ebs-frontend-aurora-nami",
"RootDeviceType": "ebs",
"OwnerId": "979382823631",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2021-05-06T15:54:43.000Z",
"Public": true,
"ImageType": "machine",
"Name": "bitnami-wordpress-5.7.1-7-r06-linux-debian-10-x86_64-hvm-ebs-frontend-aurora-nami"
},
{
"ProductCodes": [
{
"ProductCodeId": "78odkuio96rl1u75f08miu3tz",
"ProductCodeType": "marketplace"
}
],
"Description": "MXNet-1.6.0, TensorFlow-2.3.0, 2.1.0 & 1.15.3, PyTorch-1.4.0 & 1.6.0, EI, & others. NVIDIA CUDA, cuDNN, NCCL, Intel MKL-DNN, Docker, NVIDIA-Docker, Neuron,AWS Neuron,inference,AWS Inferentia,Inferentia,inf,inf1,EC2 Inf1,Elastic Inference",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"VirtualizationType": "hvm",
"Hypervisor": "xen",
"ImageOwnerAlias": "aws-marketplace",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-004566286db7148f6",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0719a5ec731c4a9b4",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 100,
"Encrypted": false
}
}
],
"Architecture": "x86_64",
"ImageLocation": "aws-marketplace/SupportedImages Deep Learning AMI (Amazon Linux 2) Version 34.0-7a4f4a31-b0bf-493c-b3fe-3d5ba59858e7-ami-036251f870bf5b53d.4",
"RootDeviceType": "ebs",
"OwnerId": "679593333241",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2020-10-06T17:09:00.000Z",
"Public": true,
"ImageType": "machine",
"Name": "SupportedImages Deep Learning AMI (Amazon Linux 2) Version 34.0-7a4f4a31-b0bf-493c-b3fe-3d5ba59858e7-ami-036251f870bf5b53d.4"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Red Hat Enterprise Linux",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-004583c651e1d7f07",
"UsageOperation": "RunInstances:0010",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/sda1",
"Ebs": {
"SnapshotId": "snap-0528a9d30401765f3",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
"ImageLocation": "979382823631/bitnami-drupal-7.67-0-r50-linux-redhat-7.5-x86_64-hvm-ebs",
"RootDeviceType": "ebs",
"OwnerId": "979382823631",
"RootDeviceName": "/dev/sda1",
"CreationDate": "2019-05-15T14:07:15.000Z",
"Public": true,
"ImageType": "machine",
"Name": "bitnami-drupal-7.67-0-r50-linux-redhat-7.5-x86_64-hvm-ebs"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"ImageId": "ami-004585220878e1c1a",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-09d73437aed0b4d2c",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
"ImageLocation": "979382823631/bitnami-osclass-3.8.0-2-r50-linux-debian-9-x86_64-hvm-ebs",
"RootDeviceType": "ebs",
"OwnerId": "979382823631",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2019-05-16T08:29:17.000Z",
"Public": true,
"ImageType": "machine",
"Name": "bitnami-osclass-3.8.0-2-r50-linux-debian-9-x86_64-hvm-ebs"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-0045895f07e3b53f1",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0653cd833b909ec7d",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
"ImageLocation": "979382823631/bitnami-ghost-3.42.7-7-r03-linux-debian-10-x86_64-hvm-ebs-nami",
"RootDeviceType": "ebs",
"OwnerId": "979382823631",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2021-12-01T05:54:42.000Z",
"Public": true,
"ImageType": "machine",
"Name": "bitnami-ghost-3.42.7-7-r03-linux-debian-10-x86_64-hvm-ebs-nami"
},
{
"VirtualizationType": "hvm",
"EnaSupport": true,
"PlatformDetails": "Linux/UNIX",
"Hypervisor": "xen",
"ImageOwnerAlias": "amazon",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-00458c099ab0e7541",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0d9dfe3e9582150fa",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 8,
"Encrypted": false
}
}
],
"Architecture": "x86_64",
"ImageLocation": "amazon/aws-elasticbeanstalk-amzn-2018.03.20.x86_64-php54-hvm-202006041836",
"RootDeviceType": "ebs",
"OwnerId": "102837901569",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2020-06-04T18:38:38.000Z",
"Public": true,
"ImageType": "machine",
"Name": "aws-elasticbeanstalk-amzn-2018.03.20.x86_64-php54-hvm-202006041836"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-0045938b357d69bb1",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0acb077985bc7898b",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
"ImageLocation": "979382823631/bitnami-wordpress-5.7.2-40-r15-linux-debian-10-x86_64-hvm-ebs-high-availability-nami",
"RootDeviceType": "ebs",
"OwnerId": "979382823631",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2021-07-16T02:16:30.000Z",
"Public": true,
"ImageType": "machine",
"Name": "bitnami-wordpress-5.7.2-40-r15-linux-debian-10-x86_64-hvm-ebs-high-availability-nami"
},
{
"VirtualizationType": "hvm",
"Description": "Cloud9 Cloud9AmazonLinux2 AMI",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"ImageOwnerAlias": "amazon",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-00459671a0ded6662",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0e3edfa566ceb76e3",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
}
],
"Architecture": "x86_64",
"ImageLocation": "amazon/Cloud9AmazonLinux2-2021-11-18T10-25",
"RootDeviceType": "ebs",
"OwnerId": "585441382316",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2021-11-18T11:12:01.000Z",
"Public": true,
"ImageType": "machine",
"Name": "Cloud9AmazonLinux2-2021-11-18T10-25"
},
{
"VirtualizationType": "hvm",
"EnaSupport": true,
"PlatformDetails": "Linux/UNIX",
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-004596f64f41c9f2c",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-03719f667299efdc0",
"DeleteOnTermination": true,
"VolumeType": "standard",
"VolumeSize": 8,
"Encrypted": false
}
}
],
"Architecture": "x86_64",
"ImageLocation": "068169053218/dremio-daasExecutor-test-274f0982-9164-4982-8b05-97e16ec143d9",
"RootDeviceType": "ebs",
"OwnerId": "068169053218",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2021-06-16T04:19:20.000Z",
"Public": true,
"ImageType": "machine",
"Name": "dremio-daasExecutor-test-274f0982-9164-4982-8b05-97e16ec143d9"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"ImageId": "ami-0045a7227274d9e5a",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-053d96d6e7b13fe4e",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
"ImageLocation": "979382823631/bitnami-tensorflow-serving-1.12.0-23-linux-debian-9-x86_64-hvm-ebs-nami",
"RootDeviceType": "ebs",
"OwnerId": "979382823631",
"RootDeviceName": "/dev/xvda",
"CreationDate": "2019-01-04T14:45:54.000Z",
"Public": true,
"ImageType": "machine",
"Name": "bitnami-tensorflow-serving-1.12.0-23-linux-debian-9-x86_64-hvm-ebs-nami"
},
{
"VirtualizationType": "hvm",
"Description": "This image may not be the latest version available and might include security vulnerabilities. Please check the latest, up-to-date, available version at https://bitnami.com/stacks.",
"PlatformDetails": "Linux/UNIX",
"EnaSupport": true,
"Hypervisor": "xen",
"State": "available",
"SriovNetSupport": "simple",
"ImageId": "ami-0045b547c0231ac6c",
"UsageOperation": "RunInstances",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"SnapshotId": "snap-0b55a356f62d464b0",
"DeleteOnTermination": true,
"VolumeType": "gp2",
"VolumeSize": 10,
"Encrypted": false
}
},
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
},
{
"DeviceName": "/dev/sdc",
"VirtualName": "ephemeral1"
},
{
"DeviceName": "/dev/sdd",
"VirtualName": "ephemeral2"
},
{
"DeviceName": "/dev/sde",
"VirtualName": "ephemeral3"
}
],
"Architecture": "x86_64",
the right answer is probably to use json ... but your second example does not seem to be valid json (or more likey is incomplete) so you can use regex...
print(re.findall('"(Name|ImageID)":\s*"(.*)"',file_contents))
Try this snippet, I'm assuming that the "long file containing the AWS AMI names" is a JSON file with a list of dictionaries. I'm assuming this because the "long version" of the file you pasted here is not properly formatted.
import json
file_name = "aws.json"
with open(file_name) as freader:
content = json.load(fp=freader)
names = []
image_ids = []
for c in content:
names.append(c["Name"])
image_ids.append(c["ImageId"])
print(names)
print(image_ids)
Outputs:
['temp-rp030-3', 'bitnami-wordpress-5.7.1-7-r06-linux-debian-10-x86_64-hvm-ebs-frontend-aurora-nami', 'SupportedImages Deep Learning AMI (Amazon Linux 2) Version 34.0-7a4f4a31-b0bf-493c-b3fe-3d5ba59858e7-ami-036251f870bf5b53d.4', 'bitnami-drupal-7.67-0-r50-linux-redhat-7.5-x86_64-hvm-ebs', 'bitnami-osclass-3.8.0-2-r50-linux-debian-9-x86_64-hvm-ebs', 'bitnami-ghost-3.42.7-7-r03-linux-debian-10-x86_64-hvm-ebs-nami', 'aws-elasticbeanstalk-amzn-2018.03.20.x86_64-php54-hvm-202006041836', 'bitnami-wordpress-5.7.2-40-r15-linux-debian-10-x86_64-hvm-ebs-high-availability-nami', 'Cloud9AmazonLinux2-2021-11-18T10-25', 'dremio-daasExecutor-test-274f0982-9164-4982-8b05-97e16ec143d9', 'bitnami-tensorflow-serving-1.12.0-23-linux-debian-9-x86_64-hvm-ebs-nami']
['ami-00455a561ee2a2221', 'ami-00455bda4a1b36dd7', 'ami-004566286db7148f6', 'ami-004583c651e1d7f07', 'ami-004585220878e1c1a', 'ami-0045895f07e3b53f1', 'ami-00458c099ab0e7541', 'ami-0045938b357d69bb1', 'ami-00459671a0ded6662', 'ami-004596f64f41c9f2c', 'ami-0045a7227274d9e5a']
If it doesn't work please update your "long version" of the file and provide a full copy of the file.
Parse the json file and get the attributes
import json
# Opening JSON file
item_list = []
f = open('/Users/Desktop/describe_images.txt')
# read JSON object
data = json.load(f)
# loop
for item in data:
item_list.append([item["Name"], item["ImageId"]])
f.close()