Mallisetty Siva Mahesh commited on
Commit
bc81c34
·
1 Parent(s): c04d620

updated s3 bucket

Browse files
Files changed (3) hide show
  1. app.py +5 -1
  2. folder.py +21 -0
  3. s3_setup.py +75 -24
app.py CHANGED
@@ -178,7 +178,11 @@ def perform_inference(file_paths: Dict[str, str], upload_to_s3: bool):
178
  if upload_to_s3:
179
  client = s3_client()
180
  bucket_name = "edgekycdocs"
181
- folder_name = f"{doc_type.split('_')[0]}docs"
 
 
 
 
182
  file_name = unprocessed_file_path.split("/")[-1].replace(" ", "_")
183
 
184
  try:
 
178
  if upload_to_s3:
179
  client = s3_client()
180
  bucket_name = "edgekycdocs"
181
+ if doc_type == "cin_llpin":
182
+ folder_name = f"{doc_type.replace('_', '')}docs"
183
+ else:
184
+ folder_name = f"{doc_type.split('_')[0]}docs"
185
+
186
  file_name = unprocessed_file_path.split("/")[-1].replace(" ", "_")
187
 
188
  try:
folder.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import boto3
3
+ # AWS credentials (if not set in environment variables or AWS CLI config)
4
+ from dotenv import load_dotenv
5
+ import os
6
+
7
+ # Load .env file
8
+ load_dotenv()
9
+ # Access variables
10
+ AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
11
+ AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
12
+ print("AWS_ACCESS_KEY_ID",AWS_ACCESS_KEY_ID)
13
+ print("AWS_SECRET_ACCESS_KEY",AWS_SECRET_ACCESS_KEY)
14
+ # Initialize S3 client
15
+
16
+ s3 = boto3.client('s3')
17
+ bucket_name = "edgekycdocs"
18
+ folder_name = "cinllpindocs"
19
+
20
+ s3.put_object(Bucket=bucket_name, Key=folder_name)
21
+ print(f"Folder '{folder_name}' created in bucket '{bucket_name}'")
s3_setup.py CHANGED
@@ -1,44 +1,95 @@
1
  import boto3
 
2
  # AWS credentials (if not set in environment variables or AWS CLI config)
3
  from dotenv import load_dotenv
4
  import os
5
  import sys
6
-
7
  from utils import doc_processing
 
 
 
 
 
8
 
9
  # Load .env file
10
  load_dotenv()
11
  # Access variables
12
  AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
13
  AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
14
- print("AWS_ACCESS_KEY_ID",AWS_ACCESS_KEY_ID)
15
- print("AWS_SECRET_ACCESS_KEY",AWS_SECRET_ACCESS_KEY)
16
  # Initialize S3 client
17
 
18
- class s3_client:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  def __init__(self):
20
- self.aws_access_key_id = AWS_ACCESS_KEY_ID
21
- self.aws_secret_access_key = AWS_SECRET_ACCESS_KEY
22
-
23
- def initialize(self):
24
- return boto3.client(
25
- 's3',
26
- aws_access_key_id=self.aws_access_key_id,
27
- aws_secret_access_key=self.aws_secret_access_key
28
- )
29
-
30
- def upload_file(self,local_file_path, bucket_name,folder_name,file_name):
31
  try:
32
- client = self.initialize()
33
- client.upload_file(local_file_path, bucket_name, f"{folder_name}/{file_name}")
34
- print(f"File uploaded successfully to {bucket_name}/{folder_name}{file_name}")
35
- url = f"https://edgekycdocs.s3.eu-north-1.amazonaws.com/{folder_name}/{file_name}"
36
- print("file url",url)
37
- return {"status": 200, "message":"file uploaded successfully" , "url" : url}
38
- except Exception as e:
39
- print("Error uploading file:", e)
40
- return {"status": 400, "message":e}
41
 
 
 
 
 
 
42
 
 
 
 
 
 
 
 
43
 
 
 
 
 
 
 
 
 
 
 
44
 
 
 
 
 
 
 
 
 
 
 
1
  import boto3
2
+
3
  # AWS credentials (if not set in environment variables or AWS CLI config)
4
  from dotenv import load_dotenv
5
  import os
6
  import sys
 
7
  from utils import doc_processing
8
+ import mimetypes
9
+ from pathlib import Path
10
+ from django.conf import settings
11
+
12
+ BASE_DIR = Path(__file__).resolve().parent.parent.parent
13
 
14
  # Load .env file
15
  load_dotenv()
16
  # Access variables
17
  AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
18
  AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
19
+ print("AWS_ACCESS_KEY_ID", AWS_ACCESS_KEY_ID)
20
+ print("AWS_SECRET_ACCESS_KEY", AWS_SECRET_ACCESS_KEY)
21
  # Initialize S3 client
22
 
23
+ # class s3_client:
24
+ # def __init__(self):
25
+ # self.aws_access_key_id = AWS_ACCESS_KEY_ID
26
+ # self.aws_secret_access_key = AWS_SECRET_ACCESS_KEY
27
+
28
+ # def initialize(self):
29
+ # return boto3.client(
30
+ # 's3',
31
+ # aws_access_key_id=self.aws_access_key_id,
32
+ # aws_secret_access_key=self.aws_secret_access_key
33
+ # )
34
+
35
+ # def upload_file(self,local_file_path, bucket_name,folder_name,file_name):
36
+ # try:
37
+ # client = self.initialize()
38
+ # client.upload_file(local_file_path, bucket_name, f"{folder_name}/{file_name}")
39
+ # print(f"File uploaded successfully to {bucket_name}/{folder_name}{file_name}")
40
+ # url = f"https://edgekycdocs.s3.eu-north-1.amazonaws.com/{folder_name}/{file_name}"
41
+ # print("file url",url)
42
+ # return {"status": 200, "message":"file uploaded successfully" , "url" : url}
43
+ # except Exception as e:
44
+ # print("Error uploading file:", e)
45
+ # return {"status": 400, "message":e}
46
+
47
+
48
+ class S3Client:
49
  def __init__(self):
50
+ self.aws_access_key_id = settings.AWS_ACCESS_KEY_ID
51
+ self.aws_secret_access_key = settings.AWS_SECRET_ACCESS_KEY
52
+ self.s3_client = boto3.client(
53
+ "s3",
54
+ aws_access_key_id=self.aws_access_key_id,
55
+ aws_secret_access_key=self.aws_secret_access_key,
56
+ )
57
+
58
+ def upload_file(self, local_file_path, bucket_name, folder_name, file_name):
 
 
59
  try:
60
+ file_key = f"{folder_name}/{file_name}"
 
 
 
 
 
 
 
 
61
 
62
+ # Determine the MIME type dynamically
63
+ # Determine the correct MIME type
64
+ content_type, _ = mimetypes.guess_type(local_file_path)
65
+ if content_type is None:
66
+ content_type = "application/octet-stream" # Default fallback
67
 
68
+ # Explicitly handle common file types
69
+ if file_name.lower().endswith(
70
+ (".pdf", ".jpg", ".jpeg", ".png", ".gif", ".webp")
71
+ ):
72
+ content_type = (
73
+ mimetypes.guess_type(file_name)[0] or "application/octet-stream"
74
+ )
75
 
76
+ # Upload with correct Content-Type
77
+ self.s3_client.upload_file(
78
+ local_file_path,
79
+ bucket_name,
80
+ file_key,
81
+ ExtraArgs={
82
+ "ContentDisposition": "inline", # Ensure inline display
83
+ "ContentType": content_type,
84
+ },
85
+ )
86
 
87
+ file_url = f"https://{bucket_name}.s3.amazonaws.com/{file_key}"
88
+ print(f"Uploading {file_name} with ContentType: {content_type}")
89
+ return {
90
+ "status": 200,
91
+ "message": "File uploaded successfully",
92
+ "url": file_url,
93
+ }
94
+ except Exception as e:
95
+ return {"status": 400, "message": str(e)}