Spaces:
Sleeping
Sleeping
Merge branch 'dev' of github.com:billmarinocam/compliancecards into dev
Browse files- compliance_analysis.py +31 -33
- data_cc.yaml +2 -2
- model_cc.yaml +4 -4
compliance_analysis.py
CHANGED
@@ -156,54 +156,52 @@ def run_compliance_analysis_on_project(dispositive_variables, project_cc_yaml):
|
|
156 |
|
157 |
def run_compliance_analysis_on_data(dispositive_variables, data_cc_yaml):
|
158 |
|
|
|
159 |
if dispositive_variables['ai_project_type']["high_risk_ai_system"] == True:
|
160 |
for key in data_cc_yaml['high_risk_ai_systems']:
|
161 |
if data_cc_yaml['high_risk_ai_systems'][f'{key}']['value'] == True:
|
162 |
dispositive_variables['msg'].append(f"This high-risk AI system fails the {key} requirements under {data_cc_yaml['high_risk_ai_systems'][f'{key}']['article']}.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
163 |
|
|
|
|
|
164 |
if dispositive_variables['ai_project_type']["gpai_model"] == True:
|
165 |
-
for value in data_cc_yaml['
|
166 |
-
if data_cc_yaml['
|
167 |
dispositive_variables['msg'].append(f"")
|
168 |
|
169 |
return dispositive_variables
|
170 |
|
171 |
def run_compliance_analysis_on_model(dispositive_variables, model_cc_yaml):
|
172 |
|
173 |
-
#
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
# if not value:
|
180 |
-
# msg = (f"Because of the model represented by , this high-risk AI system fails the technical documentation requirements under Article 11.")
|
181 |
-
# for key, value in data_cc_yaml['transparency_and_provision_of_information_to_deployers']:
|
182 |
-
# if not value:
|
183 |
-
# msg = (f"Because of the model represented by , this high-risk AI system fails the transparency requirements under Article 13.")
|
184 |
-
# for key, value in data_cc_yaml['accuracy_robustness_cybersecurity']:
|
185 |
-
# if not value:
|
186 |
-
# msg = (f"Because of the model represented by , this high-risk AI system fails the quality management requirements under Article 15.")
|
187 |
-
# for key, value in data_cc_yaml['quality_management_system']:
|
188 |
-
# if not value:
|
189 |
-
# msg = (f"Because of the model represented by , this high-risk AI system fails the quality management requirements under Article 17.")
|
190 |
|
191 |
-
#
|
192 |
-
|
193 |
-
|
194 |
-
|
|
|
|
|
195 |
|
196 |
-
|
197 |
-
|
198 |
-
|
|
|
|
|
|
|
199 |
|
200 |
-
# # TODO: No matter where we land with an orchestrator function, this function must also check to the value that has been set for both
|
201 |
-
# # GPAI models with and without systemic risk and then check to see if the relevant requirements have met if either of these values applies.
|
202 |
-
# # Right now it is only checking high-risk AI system requirements. Another thing that we likely have to add here is the cross-comparison of the
|
203 |
-
# # intended purposes. That might look like this:
|
204 |
-
# # if model_cc_yaml['intended_purpose'] not in intended_purposes:
|
205 |
-
# # return false
|
206 |
-
|
207 |
return dispositive_variables
|
208 |
|
209 |
def check_intended_purpose(dispositive_variables, project_cc, other_cc):
|
|
|
156 |
|
157 |
def run_compliance_analysis_on_data(dispositive_variables, data_cc_yaml):
|
158 |
|
159 |
+
<<<<<<< HEAD
|
160 |
if dispositive_variables['ai_project_type']["high_risk_ai_system"] == True:
|
161 |
for key in data_cc_yaml['high_risk_ai_systems']:
|
162 |
if data_cc_yaml['high_risk_ai_systems'][f'{key}']['value'] == True:
|
163 |
dispositive_variables['msg'].append(f"This high-risk AI system fails the {key} requirements under {data_cc_yaml['high_risk_ai_systems'][f'{key}']['article']}.")
|
164 |
+
=======
|
165 |
+
# If project involves a high-risk AI system, then make sure all the relevant data requirements are met (relevant attributes are positive)
|
166 |
+
|
167 |
+
if dispositive_variables['ai_project_type']["high_risk_ai_system"] == True:
|
168 |
+
for value in data_cc_yaml['high_risk_ai_system_requirements']:
|
169 |
+
if data_cc_yaml['high_risk_ai_system_requirements'][f'{value}'] == True:
|
170 |
+
dispositive_variables['msg'].append(f"")
|
171 |
+
>>>>>>> fbbf69b255ab44b93558b0d31b02a679007bc061
|
172 |
|
173 |
+
# If project involves a GPAI model, then make sure all the relevant data requirements are met (relevant attributes are positive)
|
174 |
+
|
175 |
if dispositive_variables['ai_project_type']["gpai_model"] == True:
|
176 |
+
for value in data_cc_yaml['gpai_model_requirements']:
|
177 |
+
if data_cc_yaml['gpai_model_requirements'][f'{value}'] == True: # should this be false? BM
|
178 |
dispositive_variables['msg'].append(f"")
|
179 |
|
180 |
return dispositive_variables
|
181 |
|
182 |
def run_compliance_analysis_on_model(dispositive_variables, model_cc_yaml):
|
183 |
|
184 |
+
# If project involves a high-risk AI system, then make sure all the relevant model requirements are met (relevant attributes are positive)
|
185 |
+
|
186 |
+
if dispositive_variables['ai_project_type']["high_risk_ai_system"] == True:
|
187 |
+
for value in model_cc_yaml['high_risk_ai_system_requirements']:
|
188 |
+
if model_cc_yaml['high_risk_ai_system_requirements'][f'{value}'] == True:
|
189 |
+
dispositive_variables['msg'].append(f"")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
190 |
|
191 |
+
# If project involves a GPAI model, then make sure all the relevant model requirements are met (relevant attributes are positive)
|
192 |
+
|
193 |
+
if dispositive_variables['ai_project_type']["gpai_model"] == True:
|
194 |
+
for value in model_cc_yaml['gpai_model_requirements']:
|
195 |
+
if model_cc_yaml['gpai_model_requirements'][f'{value}'] == True:
|
196 |
+
dispositive_variables['msg'].append(f"")
|
197 |
|
198 |
+
# If the GPAI model additionally carries systemic risk, then make sure all the relevant model requirements are met (relevant attributes are positive)
|
199 |
+
|
200 |
+
if dispositive_variables['ai_project_type']["gpai_model_systemic_risk"] == True:
|
201 |
+
for value in model_cc_yaml['gpai_model_with_systemic_risk_requirements']:
|
202 |
+
if model_cc_yaml['gpai_model_requirements'][f'{value}'] == True:
|
203 |
+
dispositive_variables['msg'].append(f"")
|
204 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
205 |
return dispositive_variables
|
206 |
|
207 |
def check_intended_purpose(dispositive_variables, project_cc, other_cc):
|
data_cc.yaml
CHANGED
@@ -58,7 +58,7 @@ intended_purpose:
|
|
58 |
|
59 |
# Metadata related to data-related requirements for high-risk AI systems
|
60 |
|
61 |
-
|
62 |
data_and_data_governance_data_governance:
|
63 |
article: 'Art. 10(1)-(2)'
|
64 |
verbose: 'The dataset was subject to data governance and management practices appropriate to the intended use case'
|
@@ -194,7 +194,7 @@ high_risk_ai_systems:
|
|
194 |
|
195 |
# Meta data related to data-relaed requirements for GPAI models
|
196 |
|
197 |
-
|
198 |
data_type:
|
199 |
article: 'Art. 53(1); Annex XI(2)(c)'
|
200 |
verbose: 'Documentation for the dataset is available that contains the type of data'
|
|
|
58 |
|
59 |
# Metadata related to data-related requirements for high-risk AI systems
|
60 |
|
61 |
+
high_risk_ai_system_requirements:
|
62 |
data_and_data_governance_data_governance:
|
63 |
article: 'Art. 10(1)-(2)'
|
64 |
verbose: 'The dataset was subject to data governance and management practices appropriate to the intended use case'
|
|
|
194 |
|
195 |
# Meta data related to data-relaed requirements for GPAI models
|
196 |
|
197 |
+
gpai_model_requirements:
|
198 |
data_type:
|
199 |
article: 'Art. 53(1); Annex XI(2)(c)'
|
200 |
verbose: 'Documentation for the dataset is available that contains the type of data'
|
model_cc.yaml
CHANGED
@@ -70,7 +70,7 @@ classification_of_gpai_models:
|
|
70 |
|
71 |
# Metadata related to model-related requirements for high-risk AI systems
|
72 |
|
73 |
-
|
74 |
risk_management_system_general:
|
75 |
article: 'Art. 9(2)'
|
76 |
verbose: 'A risk management system has been planned, run, reviewed, and updated throughout the model lifecycle'
|
@@ -210,7 +210,7 @@ high_risk_ai_models:
|
|
210 |
|
211 |
# Meta data related to model-related requirements for GPAI models
|
212 |
|
213 |
-
|
214 |
task:
|
215 |
article: 'Art. 53; Annex XI(1)(1)(a)'
|
216 |
verbose: 'The tasks that the model is intended to perform and the type and nature of AI systems in which it can be integrated'
|
@@ -296,7 +296,7 @@ obligations_for_providers_of_gpai_models:
|
|
296 |
verbose: 'Where applicable, detailed description of the measures put in place for the purpose of conducting internal and/or external adversarial testing (e.g. red teaming), model adaptations, including alignment and fine-tuning'
|
297 |
value: !!bool false
|
298 |
|
299 |
-
|
300 |
evaluation:
|
301 |
article: 'Art. 55(1)(a)'
|
302 |
verbose: 'Perform model evaluation in accordance with standardised protocols and tools reflecting the state of the art, including conducting and documenting adversarial testing of the model with a view to identify and mitigate systemic risk'
|
@@ -312,7 +312,7 @@ obligations_for_providers_of_gpai_models_with_systemic_risk:
|
|
312 |
|
313 |
# Metadata related to transparency requirements for some AI systems (Article 50)
|
314 |
|
315 |
-
|
316 |
generates_media: # clean this up
|
317 |
article: 'Art. 50(2)'
|
318 |
verbose: 'If the model generates synthetic audio, image, video or text content'
|
|
|
70 |
|
71 |
# Metadata related to model-related requirements for high-risk AI systems
|
72 |
|
73 |
+
high_risk_ai_system_requirements:
|
74 |
risk_management_system_general:
|
75 |
article: 'Art. 9(2)'
|
76 |
verbose: 'A risk management system has been planned, run, reviewed, and updated throughout the model lifecycle'
|
|
|
210 |
|
211 |
# Meta data related to model-related requirements for GPAI models
|
212 |
|
213 |
+
gpai_model_requirements:
|
214 |
task:
|
215 |
article: 'Art. 53; Annex XI(1)(1)(a)'
|
216 |
verbose: 'The tasks that the model is intended to perform and the type and nature of AI systems in which it can be integrated'
|
|
|
296 |
verbose: 'Where applicable, detailed description of the measures put in place for the purpose of conducting internal and/or external adversarial testing (e.g. red teaming), model adaptations, including alignment and fine-tuning'
|
297 |
value: !!bool false
|
298 |
|
299 |
+
gpai_model_with_systemic_risk_requirements:
|
300 |
evaluation:
|
301 |
article: 'Art. 55(1)(a)'
|
302 |
verbose: 'Perform model evaluation in accordance with standardised protocols and tools reflecting the state of the art, including conducting and documenting adversarial testing of the model with a view to identify and mitigate systemic risk'
|
|
|
312 |
|
313 |
# Metadata related to transparency requirements for some AI systems (Article 50)
|
314 |
|
315 |
+
transparency_requirements:
|
316 |
generates_media: # clean this up
|
317 |
article: 'Art. 50(2)'
|
318 |
verbose: 'If the model generates synthetic audio, image, video or text content'
|