From 9cf99ec78daa0990aa3da64a2f90df29eadf126a Mon Sep 17 00:00:00 2001 From: Sudo-JHare Date: Tue, 12 Aug 2025 21:55:28 +1000 Subject: [PATCH] updates --- app.py | 3 +- services.py | 65 ++++++++++++++-------------- templates/retrieve_split_data.html | 68 +++++++++++------------------- 3 files changed, 58 insertions(+), 78 deletions(-) diff --git a/app.py b/app.py index db887ca..ea89649 100644 --- a/app.py +++ b/app.py @@ -1905,7 +1905,7 @@ def proxy_hapi(subpath): except Exception as e: logger.error(f"Unexpected proxy error for {final_url}: {str(e)}", exc_info=True) return jsonify({'resourceType': 'OperationOutcome', 'issue': [{'severity': 'error', 'code': 'exception', 'diagnostics': 'An unexpected error occurred within the FHIR proxy.', 'details': {'text': str(e)}}]}), 500 -# --- End of corrected proxy_hapi function --- + # --- End of corrected proxy_hapi function --- @app.route('/api/load-ig-to-hapi', methods=['POST']) @@ -2338,6 +2338,7 @@ def retrieve_split_data(): now=datetime.datetime.now(), app_mode=app.config['APP_MODE'], api_key=app.config['API_KEY']) + @app.route('/api/retrieve-bundles', methods=['POST']) @csrf.exempt @swag_from({ diff --git a/services.py b/services.py index 7c3bd52..c494d1c 100644 --- a/services.py +++ b/services.py @@ -4583,23 +4583,21 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references else: yield json.dumps({"type": "info", "message": "Reference fetching OFF"}) + "\n" - # Determine Base URL and Headers for Proxy - base_proxy_url = f"{current_app.config['APP_BASE_URL'].rstrip('/')}/fhir" + # Determine Base URL and Headers for the requests. headers = {'Accept': 'application/fhir+json, application/fhir+xml;q=0.9, */*;q=0.8'} is_custom_url = fhir_server_url != '/fhir' and fhir_server_url is not None and fhir_server_url.startswith('http') + + final_base_url = fhir_server_url.rstrip('/') if fhir_server_url else '/fhir' + if is_custom_url: - # NEW: Add the custom URL as a query parameter for the proxy to use. - # This bypasses issues where reverse proxies might strip custom headers. - base_proxy_url = f"{base_proxy_url}?proxy-target={fhir_server_url.rstrip('/')}" - if auth_type in ['bearer', 'basic'] and auth_token: auth_display = 'Basic ' if auth_type == 'basic' else (auth_token[:10] + '...' if len(auth_token) > 10 else auth_token) yield json.dumps({"type": "info", "message": f"Using {auth_type} auth with header: Authorization: {auth_display}"}) + "\n" headers['Authorization'] = auth_token else: yield json.dumps({"type": "info", "message": "Using no authentication for custom URL"}) + "\n" - logger.debug(f"Will use proxy with proxy-target: {fhir_server_url}") + logger.debug(f"Will send requests directly to: {final_base_url}") else: yield json.dumps({"type": "info", "message": "Using no authentication for local HAPI server"}) + "\n" logger.debug("Will use proxy targeting local HAPI server") @@ -4607,25 +4605,24 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references # Fetch Initial Bundles initial_bundle_files = [] for resource_type in resources: - #url = f"{base_proxy_url}/{quote(resource_type)}" - url = urljoin(base_proxy_url, quote(resource_type)) - yield json.dumps({"type": "progress", "message": f"Fetching bundle for {resource_type} via proxy..."}) + "\n" - logger.debug(f"Sending GET request to proxy {url} with headers: {json.dumps(headers)}") + url = f"{final_base_url}/{quote(resource_type)}" + yield json.dumps({"type": "progress", "message": f"Fetching bundle for {resource_type}..."}) + "\n" + logger.debug(f"Sending GET request to {url} with headers: {json.dumps(headers)}") try: response = requests.get(url, headers=headers, timeout=60) - logger.debug(f"Proxy response for {resource_type}: HTTP {response.status_code}") + logger.debug(f"Response for {resource_type}: HTTP {response.status_code}") if response.status_code != 200: - error_detail = f"Proxy returned HTTP {response.status_code}." + error_detail = f"Server returned HTTP {response.status_code}." try: error_detail += f" Body: {response.text[:200]}..." except: pass yield json.dumps({"type": "error", "message": f"Failed to fetch {resource_type}: {error_detail}"}) + "\n" - logger.error(f"Failed to fetch {resource_type} via proxy {url}: {error_detail}") + logger.error(f"Failed to fetch {resource_type} at {url}: {error_detail}") continue try: bundle = response.json() except ValueError as e: yield json.dumps({"type": "error", "message": f"Invalid JSON response for {resource_type}: {str(e)}"}) + "\n" - logger.error(f"Invalid JSON from proxy for {resource_type} at {url}: {e}, Response: {response.text[:500]}") + logger.error(f"Invalid JSON from {resource_type} at {url}: {e}, Response: {response.text[:500]}") continue if not isinstance(bundle, dict) or bundle.get('resourceType') != 'Bundle': yield json.dumps({"type": "error", "message": f"Expected Bundle for {resource_type}, got {bundle.get('resourceType', 'unknown')}"}) + "\n" @@ -4648,8 +4645,8 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references logger.error(f"Failed to write bundle file {output_file}: {e}") continue except requests.RequestException as e: - yield json.dumps({"type": "error", "message": f"Error connecting to proxy for {resource_type}: {str(e)}"}) + "\n" - logger.error(f"Error retrieving bundle for {resource_type} via proxy {url}: {e}") + yield json.dumps({"type": "error", "message": f"Error connecting to server for {resource_type}: {str(e)}"}) + "\n" + logger.error(f"Error retrieving bundle for {resource_type} via {url}: {e}") continue except Exception as e: yield json.dumps({"type": "error", "message": f"Unexpected error fetching {resource_type}: {str(e)}"}) + "\n" @@ -4699,18 +4696,18 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references if ref_type in retrieved_references_or_types: continue - url = f"{base_proxy_url}/{quote(ref_type)}" - yield json.dumps({"type": "progress", "message": f"Fetching full bundle for type {ref_type} via proxy..."}) + "\n" - logger.debug(f"Sending GET request for full type bundle {ref_type} to proxy {url} with headers: {json.dumps(headers)}") + url = f"{final_base_url}/{quote(ref_type)}" + yield json.dumps({"type": "progress", "message": f"Fetching full bundle for type {ref_type}..."}) + "\n" + logger.debug(f"Sending GET request for full type bundle {ref_type} to {url} with headers: {json.dumps(headers)}") try: response = requests.get(url, headers=headers, timeout=180) - logger.debug(f"Proxy response for {ref_type} bundle: HTTP {response.status_code}") + logger.debug(f"Response for {ref_type} bundle: HTTP {response.status_code}") if response.status_code != 200: - error_detail = f"Proxy returned HTTP {response.status_code}." + error_detail = f"Server returned HTTP {response.status_code}." try: error_detail += f" Body: {response.text[:200]}..." except: pass yield json.dumps({"type": "warning", "message": f"Failed to fetch full bundle for {ref_type}: {error_detail}"}) + "\n" - logger.warning(f"Failed to fetch full bundle {ref_type} via proxy {url}: {error_detail}") + logger.warning(f"Failed to fetch full bundle {ref_type} via {url}: {error_detail}") retrieved_references_or_types.add(ref_type) continue @@ -4718,7 +4715,7 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references bundle = response.json() except ValueError as e: yield json.dumps({"type": "warning", "message": f"Invalid JSON for full {ref_type} bundle: {str(e)}"}) + "\n" - logger.warning(f"Invalid JSON response from proxy for full {ref_type} bundle at {url}: {e}") + logger.warning(f"Invalid JSON response from {ref_type} bundle at {url}: {e}") retrieved_references_or_types.add(ref_type) continue @@ -4742,8 +4739,8 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references logger.error(f"Failed to write full bundle file {output_file}: {e}") retrieved_references_or_types.add(ref_type) except requests.RequestException as e: - yield json.dumps({"type": "warning", "message": f"Error connecting to proxy for full {ref_type} bundle: {str(e)}"}) + "\n" - logger.warning(f"Error retrieving full {ref_type} bundle via proxy: {e}") + yield json.dumps({"type": "warning", "message": f"Error connecting to server for full {ref_type} bundle: {str(e)}"}) + "\n" + logger.warning(f"Error retrieving full {ref_type} bundle via: {e}") retrieved_references_or_types.add(ref_type) except Exception as e: yield json.dumps({"type": "warning", "message": f"Unexpected error fetching full {ref_type} bundle: {str(e)}"}) + "\n" @@ -4765,19 +4762,19 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references ref_type, ref_id = ref_parts search_param = quote(f"_id={ref_id}") - url = f"{base_proxy_url}/{quote(ref_type)}?{search_param}" - yield json.dumps({"type": "progress", "message": f"Fetching referenced {ref_type}/{ref_id} via proxy..."}) + "\n" - logger.debug(f"Sending GET request for referenced {ref} to proxy {url} with headers: {json.dumps(headers)}") + url = f"{final_base_url}/{quote(ref_type)}?{search_param}" + yield json.dumps({"type": "progress", "message": f"Fetching referenced {ref_type}/{ref_id}..."}) + "\n" + logger.debug(f"Sending GET request for referenced {ref} to {url} with headers: {json.dumps(headers)}") response = requests.get(url, headers=headers, timeout=60) - logger.debug(f"Proxy response for referenced {ref}: HTTP {response.status_code}") + logger.debug(f"Response for referenced {ref}: HTTP {response.status_code}") if response.status_code != 200: - error_detail = f"Proxy returned HTTP {response.status_code}." + error_detail = f"Server returned HTTP {response.status_code}." try: error_detail += f" Body: {response.text[:200]}..." except: pass yield json.dumps({"type": "warning", "message": f"Failed to fetch referenced {ref}: {error_detail}"}) + "\n" - logger.warning(f"Failed to fetch referenced {ref} via proxy {url}: {error_detail}") + logger.warning(f"Failed to fetch referenced {ref} via {url}: {error_detail}") retrieved_references_or_types.add(ref) continue @@ -4785,7 +4782,7 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references bundle = response.json() except ValueError as e: yield json.dumps({"type": "warning", "message": f"Invalid JSON for referenced {ref}: {str(e)}"}) + "\n" - logger.warning(f"Invalid JSON from proxy for ref {ref} at {url}: {e}") + logger.warning(f"Invalid JSON from ref {ref} at {url}: {e}") retrieved_references_or_types.add(ref) continue @@ -4815,7 +4812,7 @@ def retrieve_bundles(fhir_server_url, resources, output_zip, validate_references retrieved_references_or_types.add(ref) except requests.RequestException as e: yield json.dumps({"type": "warning", "message": f"Network error fetching referenced {ref}: {str(e)}"}) + "\n" - logger.warning(f"Network error retrieving referenced {ref} via proxy: {e}") + logger.warning(f"Network error retrieving referenced {ref} via: {e}") retrieved_references_or_types.add(ref) except Exception as e: yield json.dumps({"type": "warning", "message": f"Unexpected error fetching referenced {ref}: {str(e)}"}) + "\n" diff --git a/templates/retrieve_split_data.html b/templates/retrieve_split_data.html index c930322..38f5960 100644 --- a/templates/retrieve_split_data.html +++ b/templates/retrieve_split_data.html @@ -27,7 +27,7 @@ {% endif %} -
+ {{ form.hidden_tag() }}
@@ -401,52 +401,35 @@ document.addEventListener('DOMContentLoaded', () => { formData.append('fetch_reference_bundles', 'false'); } - const currentFhirServerUrl = useLocalHapi ? '/fhir' : fhirServerUrlInput.value.trim(); - if (!useLocalHapi && !currentFhirServerUrl) { - alert('Custom FHIR Server URL is required.'); - fhirServerUrlInput.classList.add('is-invalid'); - retrieveButton.disabled = false; - if (spinner) spinner.style.display = 'none'; - if (icon) icon.style.display = 'inline-block'; - return; - } - formData.append('fhir_server_url', currentFhirServerUrl); + const currentFhirServerUrl = useLocalHapi ? null : fhirServerUrlInput.value.trim().replace(/\/+$/, ''); + const authType = authTypeSelect?.value; + const authHeader = (authType === 'bearer' && bearerTokenInput?.value) ? `Bearer ${bearerTokenInput.value}` + : (authType === 'basic' && usernameInput?.value && passwordInput?.value) ? `Basic ${btoa(`${usernameInput.value}:${passwordInput.value}`)}` + : null; - // Add authentication fields for custom URL - if (!useLocalHapi && authTypeSelect) { - const authType = authTypeSelect.value; - formData.append('auth_type', authType); - if (authType === 'bearer' && bearerTokenInput) { - if (!bearerTokenInput.value) { - alert('Please enter a Bearer Token.'); - retrieveButton.disabled = false; - if (spinner) spinner.style.display = 'none'; - if (icon) icon.style.display = 'inline-block'; - return; - } - formData.append('bearer_token', bearerTokenInput.value); - } else if (authType === 'basic' && usernameInput && passwordInput) { - if (!usernameInput.value || !passwordInput.value) { - alert('Please enter both Username and Password for Basic Authentication.'); - retrieveButton.disabled = false; - if (spinner) spinner.style.display = 'none'; - if (icon) icon.style.display = 'inline-block'; - return; - } - formData.append('username', usernameInput.value); - formData.append('password', passwordInput.value); - } + const url = new URL('/api/stream-retrieve', window.location.origin); + if (currentFhirServerUrl) { + url.searchParams.set('proxy-target', currentFhirServerUrl); + } + selectedResources.forEach(res => url.searchParams.append('resource_type', res)); + url.searchParams.set('validate_references', formData.get('validate_references')); + url.searchParams.set('fetch_reference_bundles', formData.get('fetch_reference_bundles')); + + const headers = { 'Accept': 'application/x-ndjson' }; + if (authHeader) { + headers['Authorization'] = authHeader; + } + if (csrfTokenInput) { + headers['X-CSRFToken'] = csrfTokenInput.value; + } + if (apiKey) { + headers['X-API-Key'] = apiKey; } - const headers = { - 'Accept': 'application/x-ndjson', - 'X-CSRFToken': csrfTokenInput ? csrfTokenInput.value : '', - 'X-API-Key': apiKey - }; - console.log(`Submitting retrieve request. Server: ${currentFhirServerUrl}, ValidateRefs: ${formData.get('validate_references')}, FetchRefBundles: ${formData.get('fetch_reference_bundles')}, AuthType: ${formData.get('auth_type')}`); + console.log(`Submitting retrieve request. URL: ${url.toString()}, Headers: ${JSON.stringify(headers)}`); try { - const response = await fetch('/api/retrieve-bundles', { method: 'POST', headers: headers, body: formData }); + const response = await fetch(url.toString(), { method: 'GET', headers: headers }); if (!response.ok) { const errorData = await response.json().catch(() => ({ message: 'Failed to parse error response.' })); @@ -454,7 +437,6 @@ document.addEventListener('DOMContentLoaded', () => { } retrieveZipPath = response.headers.get('X-Zip-Path'); console.log(`Received X-Zip-Path: ${retrieveZipPath}`); - const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = '';