From db9bb8fcfa1cee102c845a660f8b593d280ddde5 Mon Sep 17 00:00:00 2001 From: Yankyyyy Date: Tue, 16 Sep 2025 12:34:14 +0000 Subject: [PATCH 1/3] feat: Github integration with Task and Project doctype --- .../github_settings/github_settings.js | 66 +- erpnext_github_integration/github_api.py | 901 +++++++++--------- .../public/js/task_client.js | 11 +- 3 files changed, 500 insertions(+), 478 deletions(-) diff --git a/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js b/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js index 03cb860..27be1e8 100644 --- a/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js +++ b/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js @@ -141,16 +141,68 @@ frappe.ui.form.on("GitHub Settings", { // Sync All Repositories button frm.add_custom_button(__('Sync All Repositories'), function() { frappe.confirm(__('This will sync all existing repositories. Continue?'), function() { + // Prepare and show a simple progress dialog + const dlg = new frappe.ui.Dialog({ + title: __('GitHub Sync Progress'), + fields: [ + { fieldtype: 'HTML', fieldname: 'progress_html' } + ], + primary_action_label: __('Close'), + primary_action: function() { + dlg.hide(); + } + }); + + const $progress = $(` +
+
+
0%
+
+
+
+
+ `); + + dlg.fields_dict.progress_html.$wrapper.empty().append($progress); + dlg.show(); + + // realtime listener (single handler) + frappe.realtime.on('github_sync_progress', (data) => { + try { + const total = data.total || 1; + const progress = data.progress || 0; + const percent = Math.round((progress / total) * 100); + dlg.fields_dict.progress_html.$wrapper.find('.progress-bar').css('width', percent + '%').text(percent + '%'); + + const msg = data.msg || (data.repo ? (`${data.repo}: ${data.phase || ''}`) : 'Progress update'); + $('#progress_msg').text(msg); + + // optional details + let details = []; + if (data.repo) details.push(`repo: ${data.repo}`); + if (data.status) details.push(`status: ${data.status}`); + if (data.time_s) details.push(`time: ${data.time_s}s`); + if (typeof data.success !== 'undefined') details.push(`success: ${data.success}`); + if (typeof data.failed !== 'undefined') details.push(`failed: ${data.failed}`); + $('#progress_details').html(details.join(' • ')); + } catch (e) { + // ignore UI errors + console.error(e); + } + }); + + // call the server entrypoint that enqueues the job frappe.call({ - method: 'erpnext_github_integration.github_api.sync_all_repositories', + method: 'erpnext_github_integration.github_api.start_sync_all_repositories', callback: function(r) { - if (r.message) { - frappe.msgprint({ - title: __('Repositories Sync'), - indicator: r.message.failed > 0 ? 'red' : 'green', - message: __(`Success: ${r.message.success}
Failed: ${r.message.failed}`) - }); + if (r.message && r.message.status === 'queued') { + frappe.msgprint(__('Repository sync queued — watch the progress dialog.')); + } else { + frappe.msgprint(__('Repository sync started.')); } + }, + error: function(err) { + frappe.msgprint(__('Failed to start repo sync: ') + (err && err.message || '')); } }); }); diff --git a/erpnext_github_integration/github_api.py b/erpnext_github_integration/github_api.py index 53aa41d..fb7336d 100644 --- a/erpnext_github_integration/github_api.py +++ b/erpnext_github_integration/github_api.py @@ -1,58 +1,41 @@ import frappe, json from frappe import _ +from frappe.model.meta import get_table_columns import datetime +import time from datetime import datetime, timedelta from dateutil import parser import pytz +from concurrent.futures import ThreadPoolExecutor +import requests from .github_client import github_request def has_role(role): """Compatibility function for different Frappe versions""" try: - # For older Frappe versions return frappe.has_role(role) except AttributeError: - # For Frappe v15+ return role in frappe.get_roles() - -# Helper function to convert GitHub datetime to MySQL format + def convert_github_datetime(dt_string): if not dt_string: return None try: - # Parse ISO 8601 format dt = parser.parse(dt_string) - - # Ensure it's treated as UTC if no timezone info if dt.tzinfo is None: dt = pytz.utc.localize(dt) - elif dt.tzinfo.utcoffset(dt) == pytz.utc.utcoffset(dt): - pass # Already UTC - - # Convert to IST ist_tz = pytz.timezone('Asia/Kolkata') local_dt = dt.astimezone(ist_tz) - - # Return without timezone info for MySQL return local_dt.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S') - except (ValueError, TypeError) as e: frappe.log_error(f'Error parsing datetime {dt_string}: {str(e)}', 'DateTime Parse Error') return None -# Usage -# if not has_role('GitHub Admin'): -# frappe.throw("Permission required") - def _require_github_admin(): if not has_role('GitHub Admin'): frappe.throw(_('Only users with the GitHub Admin role can perform this action.')) def _can_sync_repo(repo_full_name): - """Return True if current user can sync the repo: - - GitHub Admins can always sync - - Project Manager of any Project linked to this repo can sync - """ if has_role('GitHub Admin'): return True projects = frappe.get_all('Project', filters={'repository': repo_full_name}, fields=['name', 'project_manager']) @@ -62,79 +45,150 @@ def _can_sync_repo(repo_full_name): return True return False +def _publish_progress(event='github_sync_progress', data=None, user=None): + try: + frappe.publish_realtime(event, data or {}, user=user) + except Exception: + frappe.log_error(frappe.get_traceback(), "Failed to publish realtime progress") + +def _replace_child_table_rows_multi_insert(parent_doctype, parent_name, child_table_fieldname, rows): + """ + Replace child table rows with a single multi-row INSERT (fast, DB-level). + Ensure docstatus is included and set to 0 for all child table rows. + """ + meta = frappe.get_meta(parent_doctype) + child_meta = frappe.get_meta(meta.get_field(child_table_fieldname).options) + valid_columns = child_meta.get_valid_columns() + base_columns = ["name", "owner", "creation", "modified", "modified_by", + "parent", "parentfield", "parenttype", "idx", "docstatus"] + insert_columns = base_columns + [c for c in valid_columns if c not in base_columns] + + # Delete existing rows + frappe.db.sql(f""" + DELETE FROM `tab{child_meta.name}` + WHERE parent=%s AND parentfield=%s AND parenttype=%s + """, (parent_name, child_table_fieldname, parent_doctype)) + + if not rows: + return + + now = frappe.utils.now_datetime() + values = [] + for idx, row in enumerate(rows, start=1): + base_vals = { + "name": frappe.generate_hash("", 10), + "owner": frappe.session.user, + "creation": now, + "modified": now, + "modified_by": frappe.session.user, + "parent": parent_name, + "parentfield": child_table_fieldname, + "parenttype": parent_doctype, + "idx": idx, + "docstatus": 0 # Explicitly set docstatus to 0 + } + vals = [] + for col in insert_columns: + vals.append(base_vals.get(col, row.get(col))) + values.append(vals) + + placeholders = "(" + ", ".join(["%s"] * len(insert_columns)) + ")" + insert_sql = f""" + INSERT INTO `tab{child_meta.name}` ({", ".join(f"`{c}`" for c in insert_columns)}) + VALUES {", ".join([placeholders] * len(values))} + """ + flat_values = [val for row_vals in values for val in row_vals] + frappe.db.sql(insert_sql, tuple(flat_values)) + +def fetch_paginated_data(path, token, params=None): + params = params or {} + params['per_page'] = 100 + results = [] + page = 1 + while True: + params['page'] = page + data = github_request('GET', path, token, params=params) or [] + results.extend(data) + if len(data) < 100: + break + page += 1 + return results + +def check_rate_limit(token): + response = github_request('GET', '/rate_limit', token) + remaining = response.get('rate', {}).get('remaining', 0) + reset_time = response.get('rate', {}).get('reset', 0) + if remaining < 100: + reset_dt = datetime.fromtimestamp(reset_time) + frappe.throw(f"GitHub API rate limit too low ({remaining} requests remaining). Try again after {reset_dt}.") + return remaining + +def github_request(method, path, token, params=None, data=None): + headers = {'Authorization': f'token {token}', 'Accept': 'application/vnd.github.v3+json'} + url = f"https://api.github.com{path}" + for attempt in range(3): + try: + response = requests.request(method, url, headers=headers, params=params, json=data) + if response.status_code == 429: + retry_after = int(response.headers.get('Retry-After', 60)) + time.sleep(retry_after) + continue + response.raise_for_status() + return response.json() if response.content else {} + except requests.exceptions.RequestException as e: + if attempt == 2: + raise Exception(f"GitHub API request failed: {str(e)}") + time.sleep(2 ** attempt) + return {} + @frappe.whitelist() def test_connection(): - """Test GitHub API connection""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: return {'success': False, 'error': 'GitHub Personal Access Token not configured'} - try: + check_rate_limit(token) user_info = github_request('GET', '/user', token) if user_info and user_info.get('login'): return {'success': True, 'user': user_info.get('login')} - else: - return {'success': False, 'error': 'Invalid response from GitHub API'} + return {'success': False, 'error': 'Invalid response from GitHub API'} except Exception as e: return {'success': False, 'error': str(e)} - + @frappe.whitelist() def get_github_username_by_email(email): - """Fetch GitHub username from GitHub API using email""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') - if not token: return {'success': False, 'error': 'GitHub Personal Access Token not configured'} - try: - # Use github_request to search for users by email + check_rate_limit(token) search_results = github_request('GET', f'/search/users?q={email}+in:email', token) - if search_results and search_results.get('total_count', 0) > 0 and search_results.get('items'): - # Return the first matching username return { 'success': True, 'github_username': search_results['items'][0]['login'], 'total_results': search_results['total_count'] } - else: - return { - 'success': False, - 'error': f'No GitHub user found with email: {email}' - } - + return {'success': False, 'error': f'No GitHub user found with email: {email}'} except Exception as e: return {'success': False, 'error': str(e)} - + @frappe.whitelist() def fetch_all_repositories(organization=None): - """Fetch all repositories from GitHub and create/update them in ERPNext""" try: settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') - if not token: frappe.throw('GitHub Personal Access Token not configured') - - # Get repositories from GitHub - if organization: - repos = github_request('GET', f'/orgs/{organization}/repos', token, params={'per_page': 100}) - else: - repos = github_request('GET', '/user/repos', token, params={'per_page': 100, 'affiliation': 'owner'}) - + check_rate_limit(token) + repos = fetch_paginated_data(f'/orgs/{organization}/repos' if organization else '/user/repos', token) if not repos: return {'success': False, 'message': 'No repositories found'} - - created_count = 0 - updated_count = 0 - + created_count = updated_count = 0 for repo in repos: - # Check if repository already exists repo_name = repo.get('full_name') - existing_repo = frappe.db.exists('Repository', {'full_name': repo_name}) - repo_data = { 'full_name': repo_name, 'repo_name': repo.get('name'), @@ -145,44 +199,36 @@ def fetch_all_repositories(organization=None): 'default_branch': repo.get('default_branch'), 'is_synced': 0 } - - if existing_repo: - # Update existing repository - doc = frappe.get_doc('Repository', existing_repo) + if frappe.db.exists('Repository', {'full_name': repo_name}): + doc = frappe.get_doc('Repository', {'full_name': repo_name}) doc.update(repo_data) - doc.save() + doc.save(ignore_permissions=True) updated_count += 1 - frappe.logger().info(f"Updated repository: {repo_name}") else: - # Create new repository - doc = frappe.get_doc({ - 'doctype': 'Repository', - **repo_data - }) - doc.insert() + doc = frappe.get_doc({'doctype': 'Repository', **repo_data}) + doc.insert(ignore_permissions=True) created_count += 1 - frappe.logger().info(f"Created repository: {repo_name}") - + if (created_count + updated_count) % 10 == 0: + frappe.db.commit() + frappe.db.commit() return { 'success': True, 'message': f'Successfully fetched {len(repos)} repositories. Created: {created_count}, Updated: {updated_count}' } - except Exception as e: - frappe.logger().error(f"Error fetching repositories: {str(e)}") + frappe.db.rollback() + frappe.log_error(f"Error fetching repositories: {str(e)}", "GitHub Fetch Repos") return {'success': False, 'message': f'Error: {str(e)}'} @frappe.whitelist() def get_sync_statistics(): - """Get synchronization statistics""" - stats = { + return { 'repositories': frappe.db.count('Repository'), 'issues': frappe.db.count('Repository Issue'), 'pull_requests': frappe.db.count('Repository Pull Request'), 'members': frappe.db.count('Repository Member'), 'branches': frappe.db.count('Repository Branch') } - return stats @frappe.whitelist() def can_user_sync_repo(repo_full_name): @@ -194,11 +240,9 @@ def list_repositories(organization=None): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - if organization: - path = f"/orgs/{organization}/repos" - else: - path = "/user/repos" - return github_request('GET', path, token, params={'per_page': 100}) + check_rate_limit(token) + path = f"/orgs/{organization}/repos" if organization else "/user/repos" + return fetch_paginated_data(path, token) @frappe.whitelist() def list_branches(repo_full_name, per_page=100): @@ -206,8 +250,8 @@ def list_branches(repo_full_name, per_page=100): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - path = f"/repos/{repo_full_name}/branches" - return github_request('GET', path, token, params={'per_page': per_page}) + check_rate_limit(token) + return fetch_paginated_data(f"/repos/{repo_full_name}/branches", token, params={'per_page': per_page}) @frappe.whitelist() def list_teams(org_name, per_page=100): @@ -215,8 +259,8 @@ def list_teams(org_name, per_page=100): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - path = f"/orgs/{org_name}/teams" - return github_request('GET', path, token, params={'per_page': per_page}) + check_rate_limit(token) + return fetch_paginated_data(f"/orgs/{org_name}/teams", token, params={'per_page': per_page}) @frappe.whitelist() def list_repo_members(repo_full_name, per_page=100): @@ -224,11 +268,11 @@ def list_repo_members(repo_full_name, per_page=100): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) + check_rate_limit(token) try: - members = github_request('GET', f"/repos/{repo_full_name}/collaborators", token, params={'per_page': per_page}) + return fetch_paginated_data(f"/repos/{repo_full_name}/collaborators", token, params={'per_page': per_page}) except Exception as e: frappe.throw(_('GitHub list repo members failed: {0}').format(str(e))) - return members @frappe.whitelist() def assign_issue(repo_full_name, issue_number, assignees): @@ -236,238 +280,165 @@ def assign_issue(repo_full_name, issue_number, assignees): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - - # normalize assignees input + check_rate_limit(token) if isinstance(assignees, str): try: assignees = json.loads(assignees) except Exception: assignees = [a.strip() for a in assignees.split(',') if a.strip()] - github_usernames = [] - + rows = [] try: - local = frappe.get_doc('Repository Issue', { - 'repository': repo_full_name, - 'issue_number': int(issue_number) - }) - - # Reset assignees table - local.set('assignees_table', []) - + local = frappe.get_doc('Repository Issue', {'repository': repo_full_name, 'issue_number': int(issue_number)}) for user_id in assignees: - # map ERPNext user → GitHub username gh_username = frappe.db.get_value("User", user_id, "github_username") - if not gh_username: - frappe.log_error( - f"User {user_id} has no GitHub username set", - "GitHub Assignee Mapping" - ) - continue # skip if no GitHub username - + frappe.log_error(f"User {user_id} has no GitHub username set", "GitHub Assignee Mapping") + continue github_usernames.append(gh_username) - - # store ERPNext user (email / id) in local doc - local.append('assignees_table', { - 'issue': local.name, - 'user': user_id - }) - - local.save(ignore_permissions=True) - + rows.append({'user': user_id, 'issue': local.name}) + _replace_child_table_rows_multi_insert('Repository Issue', local.name, 'assignees_table', rows) except Exception: frappe.log_error(frappe.get_traceback(), "Failed to update local Repository Issue assignees") - - # send to GitHub (GitHub API needs GitHub usernames) payload = {'assignees': github_usernames} try: - resp = github_request( - 'PATCH', - f"/repos/{repo_full_name}/issues/{issue_number}", - token, - data=payload - ) + resp = github_request('PATCH', f"/repos/{repo_full_name}/issues/{issue_number}", token, data=payload) + return resp except Exception as e: frappe.throw(_('GitHub assign issue failed: {0}').format(str(e))) - return resp - @frappe.whitelist() def add_pr_reviewer(repo_full_name, pr_number, reviewers): settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - - # normalize reviewers input + check_rate_limit(token) if isinstance(reviewers, str): try: reviewers = json.loads(reviewers) except Exception: reviewers = [r.strip() for r in reviewers.split(',') if r.strip()] - github_usernames = [] - + rows = [] try: - local = frappe.get_doc('Repository Pull Request', { - 'repository': repo_full_name, - 'pr_number': int(pr_number) - }) - - # Reset reviewers table - local.set('reviewers_table', []) - + local = frappe.get_doc('Repository Pull Request', {'repository': repo_full_name, 'pr_number': int(pr_number)}) for user_id in reviewers: - # map ERPNext user → GitHub username gh_username = frappe.db.get_value("User", user_id, "github_username") - if not gh_username: - frappe.log_error( - f"User {user_id} has no GitHub username set", - "GitHub Reviewer Mapping" - ) - continue # skip if no GitHub username - + frappe.log_error(f"User {user_id} has no GitHub username set", "GitHub Reviewer Mapping") + continue github_usernames.append(gh_username) - - # store ERPNext user (email / id) in local doc - local.append('reviewers_table', { - 'pull_request': local.name, - 'user': user_id - }) - - local.save(ignore_permissions=True) - + rows.append({'user': user_id, 'pull_request': local.name}) + _replace_child_table_rows_multi_insert('Repository Pull Request', local.name, 'reviewers_table', rows) except Exception: frappe.log_error(frappe.get_traceback(), "Failed to update local Repository Pull Request reviewers") - - # send to GitHub (GitHub API needs GitHub usernames) payload = {'reviewers': github_usernames} try: - resp = github_request( - 'POST', - f"/repos/{repo_full_name}/pulls/{pr_number}/requested_reviewers", - token, - data=payload - ) + resp = github_request('POST', f"/repos/{repo_full_name}/pulls/{pr_number}/requested_reviewers", token, data=payload) + return resp except Exception as e: frappe.throw(_('GitHub add PR reviewer failed: {0}').format(str(e))) - return resp - @frappe.whitelist() -def sync_repo(repository): +def sync_repo(repository, user=None): repo_full = repository + _publish_progress(data={'repo': repo_full, 'phase': 'start', 'msg': f'Starting sync for {repo_full}'}, user=user) if not _can_sync_repo(repo_full): frappe.throw(_('You do not have permission to sync this repository.')) settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - - # Get repository info - repo_info = github_request('GET', f'/repos/{repo_full}', token) or {} - branches = github_request('GET', f'/repos/{repo_full}/branches', token) or [] - issues = github_request('GET', f'/repos/{repo_full}/issues', token, params={'state':'all'}) or [] - pulls = github_request('GET', f'/repos/{repo_full}/pulls', token, params={'state':'all'}) or [] - members = github_request('GET', f'/repos/{repo_full}/collaborators', token) or [] - - # Upsert repo doc + check_rate_limit(token) try: - repo_doc = frappe.get_doc('Repository', {'full_name': repo_full}) - repo_doc.is_synced = 1 - repo_doc.last_synced = frappe.utils.now() - repo_doc.github_id = str(repo_info.get('id', '')) - repo_doc.visibility = 'Private' if repo_info.get('private') else 'Public' - repo_doc.default_branch = repo_info.get('default_branch', 'main') - repo_doc.save(ignore_permissions=True) - except frappe.DoesNotExistError: - repo_doc = frappe.get_doc({ + # Initialize or fetch repository document + repo_doc = frappe.get_doc('Repository', {'full_name': repo_full}) if frappe.db.exists('Repository', {'full_name': repo_full}) else frappe.get_doc({ 'doctype': 'Repository', 'full_name': repo_full, 'repo_name': repo_full.split('/')[-1], 'repo_owner': repo_full.split('/')[0], - 'url': f'https://github.com/{repo_full}', + 'url': f'https://github.com/{repo_full}' + }) + last_synced = repo_doc.last_synced + params = {'state': 'all', 'per_page': 100} + if last_synced: + params['since'] = last_synced.isoformat() + + # Parallel API calls + def fetch_repo_info(): + return github_request('GET', f'/repos/{repo_full}', token) or {} + def fetch_branches(): + return fetch_paginated_data(f'/repos/{repo_full}/branches', token) + def fetch_issues(): + return fetch_paginated_data(f'/repos/{repo_full}/issues', token, params=params) + def fetch_pulls(): + return fetch_paginated_data(f'/repos/{repo_full}/pulls', token, params=params) + def fetch_members(): + return fetch_paginated_data(f'/repos/{repo_full}/collaborators', token) + + with ThreadPoolExecutor(max_workers=4) as executor: + futures = { + 'repo_info': executor.submit(fetch_repo_info), + 'branches': executor.submit(fetch_branches), + 'issues': executor.submit(fetch_issues), + 'pulls': executor.submit(fetch_pulls), + 'members': executor.submit(fetch_members) + } + repo_info = futures['repo_info'].result() + branches = futures['branches'].result() + issues = futures['issues'].result() + pulls = futures['pulls'].result() + members = futures['members'].result() + + # Update repository fields + repo_doc.update({ 'github_id': str(repo_info.get('id', '')), 'visibility': 'Private' if repo_info.get('private') else 'Public', 'default_branch': repo_info.get('default_branch', 'main'), 'is_synced': 1, 'last_synced': frappe.utils.now() }) - repo_doc.insert(ignore_permissions=True) - - # Clear and update branches - repo_doc.set('branches_table', []) - for b in branches: - branch_name = b.get('name') - - # Get the latest commit for this specific branch (this includes full commit data) - commits = github_request('GET', f'/repos/{repo_full}/commits', token, - params={'sha': branch_name, 'per_page': 1}) or [] - - if commits: - latest_commit = commits[0] - # This should have the full commit data including dates - commit_date_str = latest_commit.get('commit', {}).get('author', {}).get('date') - if commit_date_str: - commit_date = convert_github_datetime(commit_date_str) - frappe.log_error(f'Branch: {branch_name}, Latest Commit Date: {commit_date}', 'GitHub Sync Debug') - repo_doc.append('branches_table', { - 'repo_full_name': repo_full, - 'branch_name': b.get('name'), - 'commit_sha': b.get('commit', {}).get('sha'), - 'protected': b.get('protected', False), - 'last_updated': commit_date or '' - }) - - # Clear and update members - repo_doc.set('members_table', []) - for m in members: - user_info = github_request('GET', f"/users/{m.get('login')}", token) - m_email = user_info.get("email") or "" - repo_doc.append('members_table', { - 'repo_full_name': repo_full, - 'github_username': m.get('login'), - 'github_id': str(m.get('id', '')), - 'role': 'maintainer' if m.get('permissions', {}).get('admin') else 'member', - 'email': m_email or '' - }) - - repo_doc.save(ignore_permissions=True) - - # Sync issues - for issue in issues: - if issue.get('pull_request'): - continue # Skip pull requests (they're handled separately) - - # Check if issue exists - issue_filters = {'repository': repo_full, 'issue_number': issue.get('number')} - existing_issue = frappe.db.exists('Repository Issue', issue_filters) - - if existing_issue: - # Update existing issue - local = frappe.get_doc('Repository Issue', issue_filters) - local.title = issue.get('title') - local.body = issue.get('body') or '' - local.state = issue.get('state') - local.labels = ','.join([lab.get('name') for lab in issue.get('labels', [])]) - local.url = issue.get('html_url') - local.github_id = str(issue.get('id', '')) - local.updated_at = convert_github_datetime(issue.get('updated_at')) - - # Update assignees - local.set('assignees_table', []) - for assignee in issue.get('assignees', []): - local.append('assignees_table', { - 'user': assignee.get('login') - }) - - local.save(ignore_permissions=True) - else: - # Create new issue - issue_doc = frappe.get_doc({ - 'doctype': 'Repository Issue', + + # Prepare branches + branch_rows = [] + for b in branches: + commits = github_request('GET', f'/repos/{repo_full}/commits', token, params={'sha': b.get('name'), 'per_page': 1}) or [] + commit_date = convert_github_datetime(commits[0].get('commit', {}).get('author', {}).get('date')) if commits else None + branch_rows.append({ + 'repo_full_name': repo_full, + 'branch_name': b.get('name'), + 'commit_sha': b.get('commit', {}).get('sha'), + 'protected': b.get('protected', False), + 'last_updated': commit_date or '' + }) + + # Prepare members + member_rows = [] + for m in members: + user_info = github_request('GET', f"/users/{m.get('login')}", token) or {} + member_rows.append({ + 'repo_full_name': repo_full, + 'github_username': m.get('login'), + 'github_id': str(m.get('id', '')), + 'role': 'maintainer' if m.get('permissions', {}).get('admin') else 'member', + 'email': user_info.get('email') or '' + }) + + # Update child tables for Repository + _replace_child_table_rows_multi_insert('Repository', repo_doc.name, 'branches_table', branch_rows) + _replace_child_table_rows_multi_insert('Repository', repo_doc.name, 'members_table', member_rows) + repo_doc.save(ignore_permissions=True) + + # Handle issues + issue_rows = [] + issue_assignee_rows = {} + for issue in issues: + if issue.get('pull_request'): + continue + issue_name = frappe.generate_hash("", 10) + issue_rows.append({ + 'name': issue_name, 'repository': repo_full, 'issue_number': issue.get('number'), 'title': issue.get('title'), @@ -479,47 +450,33 @@ def sync_repo(repository): 'created_at': convert_github_datetime(issue.get('created_at')), 'updated_at': convert_github_datetime(issue.get('updated_at')) }) - - # Add assignees - for assignee in issue.get('assignees', []): - issue_doc.append('assignees_table', { - 'user': assignee.get('login') - }) - - issue_doc.insert(ignore_permissions=True) - - # Sync pull requests - for pr in pulls: - # Check if PR exists - pr_filters = {'repository': repo_full, 'pr_number': pr.get('number')} - existing_pr = frappe.db.exists('Repository Pull Request', pr_filters) - - if existing_pr: - # Update existing PR - local = frappe.get_doc('Repository Pull Request', pr_filters) - local.title = pr.get('title') - local.body = pr.get('body') or '' - local.state = pr.get('state') - local.head_branch = pr.get('head', {}).get('ref') - local.base_branch = pr.get('base', {}).get('ref') - local.author = pr.get('user', {}).get('login') - local.mergeable_state = pr.get('mergeable_state') - local.github_id = str(pr.get('id', '')) - local.url = pr.get('html_url') - local.updated_at = convert_github_datetime(pr.get('updated_at')) - - # Update reviewers - local.set('reviewers_table', []) - for reviewer in pr.get('requested_reviewers', []): - local.append('reviewers_table', { - 'user': reviewer.get('login') + issue_assignee_rows[issue_name] = [ + {'user': frappe.db.get_value("User", {"github_username": assignee.get('login')}, "name") or assignee.get('login'), 'issue': issue_name} + for assignee in issue.get('assignees', []) + ] + + if issue_rows: + # Delete existing issues to avoid duplicates + frappe.db.sql(f""" + DELETE FROM `tabRepository Issue` + WHERE repository=%s AND issue_number IN %s + """, (repo_full, [row['issue_number'] for row in issue_rows])) + for issue_row in issue_rows: + issue_doc = frappe.get_doc({ + 'doctype': 'Repository Issue', + **issue_row }) - - local.save(ignore_permissions=True) - else: - # Create new PR - pr_doc = frappe.get_doc({ - 'doctype': 'Repository Pull Request', + issue_doc.insert(ignore_permissions=True) + if issue_name in issue_assignee_rows: + _replace_child_table_rows_multi_insert('Repository Issue', issue_row['name'], 'assignees_table', issue_assignee_rows[issue_row['name']]) + + # Handle pull requests + pr_rows = [] + pr_reviewer_rows = {} + for pr in pulls: + pr_name = frappe.generate_hash("", 10) + pr_rows.append({ + 'name': pr_name, 'repository': repo_full, 'pr_number': pr.get('number'), 'title': pr.get('title'), @@ -534,23 +491,44 @@ def sync_repo(repository): 'created_at': convert_github_datetime(pr.get('created_at')), 'updated_at': convert_github_datetime(pr.get('updated_at')) }) - - # Add reviewers - for reviewer in pr.get('requested_reviewers', []): - pr_doc.append('reviewers_table', { - 'user': reviewer.get('login') + pr_reviewer_rows[pr_name] = [ + {'user': frappe.db.get_value("User", {"github_username": reviewer.get('login')}, "name") or reviewer.get('login'), 'pull_request': pr_name} + for reviewer in pr.get('requested_reviewers', []) + ] + + if pr_rows: + # Delete existing pull requests to avoid duplicates + frappe.db.sql(f""" + DELETE FROM `tabRepository Pull Request` + WHERE repository=%s AND pr_number IN %s + """, (repo_full, [row['pr_number'] for row in pr_rows])) + for pr_row in pr_rows: + pr_doc = frappe.get_doc({ + 'doctype': 'Repository Pull Request', + **pr_row }) - - pr_doc.insert(ignore_permissions=True) - - return { - 'success': True, - 'message': f'Synced repository {repo_full}', - 'branches': len(branches), - 'issues': len(issues), - 'pulls': len(pulls), - 'members': len(members) - } + pr_doc.insert(ignore_permissions=True) + if pr_row['name'] in pr_reviewer_rows: + _replace_child_table_rows_multi_insert('Repository Pull Request', pr_row['name'], 'reviewers_table', pr_reviewer_rows[pr_row['name']]) + + frappe.db.commit() + + if len(issues) >= 100 or len(pulls) >= 100: + _publish_progress(data={'repo': repo_full, 'phase': 'done', 'msg': f'Sync complete for {repo_full}', 'issues': len([i for i in issues if not i.get('pull_request')]), 'pulls': len(pulls)}, user=user) + + return { + 'success': True, + 'message': f'Synced repository {repo_full}', + 'branches': len(branches), + 'issues': len([i for i in issues if not i.get('pull_request')]), + 'pulls': len(pulls), + 'members': len(members) + } + except Exception as e: + frappe.db.rollback() + frappe.log_error(f"Error syncing repository {repo_full}: {str(e)}", "GitHub Sync Error") + _publish_progress(data={'repo': repo_full, 'phase': 'error', 'msg': str(e)}, user=user) + raise e @frappe.whitelist() def create_issue(repository, title, body=None, assignees=None, labels=None): @@ -558,9 +536,9 @@ def create_issue(repository, title, body=None, assignees=None, labels=None): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - + check_rate_limit(token) payload = {'title': title} - if body: + if body: payload['body'] = body if assignees: if isinstance(assignees, str): @@ -576,12 +554,13 @@ def create_issue(repository, title, body=None, assignees=None, labels=None): except Exception: labels = [l.strip() for l in labels.split(',') if l.strip()] payload['labels'] = labels - resp = github_request('POST', f'/repos/{repository}/issues', token, data=payload) if resp: try: + issue_name = frappe.generate_hash("", 10) doc = frappe.get_doc({ 'doctype': 'Repository Issue', + 'name': issue_name, 'repository': repository, 'issue_number': resp.get('number'), 'title': resp.get('title'), @@ -593,39 +572,39 @@ def create_issue(repository, title, body=None, assignees=None, labels=None): 'created_at': convert_github_datetime(resp.get('created_at')), 'updated_at': convert_github_datetime(resp.get('updated_at')) }) - - # Add assignees - for assignee in resp.get('assignees', []): - doc.append('assignees_table', { - 'user': assignee.get('login') - }) - doc.insert(ignore_permissions=True) + rows = [ + {'user': frappe.db.get_value("User", {"github_username": assignee.get('login')}, "name") or assignee.get('login'), 'issue': issue_name} + for assignee in resp.get('assignees', []) + ] + _replace_child_table_rows_multi_insert('Repository Issue', issue_name, 'assignees_table', rows) + frappe.db.commit() return {'issue': resp, 'local_doc': doc.name} - except Exception: - pass + except Exception as e: + frappe.db.rollback() + frappe.log_error(f"Error creating local issue: {str(e)}", "GitHub Create Issue") return resp @frappe.whitelist() def bulk_create_issues(repository, issues): - """Bulk create multiple issues in a repository""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - + check_rate_limit(token) if isinstance(issues, str): issues = json.loads(issues) - created_issues = [] - for issue_data in issues: + issue_rows = [] + issue_assignee_rows = {} + for i, issue_data in enumerate(issues): try: resp = github_request('POST', f'/repos/{repository}/issues', token, data=issue_data) if resp: created_issues.append(resp) - # Create local record - doc = frappe.get_doc({ - 'doctype': 'Repository Issue', + issue_name = frappe.generate_hash("", 10) + issue_rows.append({ + 'name': issue_name, 'repository': repository, 'issue_number': resp.get('number'), 'title': resp.get('title'), @@ -636,10 +615,20 @@ def bulk_create_issues(repository, issues): 'created_at': convert_github_datetime(resp.get('created_at')), 'updated_at': convert_github_datetime(resp.get('updated_at')) }) - doc.insert(ignore_permissions=True) + issue_assignee_rows[issue_name] = [ + {'user': frappe.db.get_value("User", {"github_username": assignee.get('login')}, "name") or assignee.get('login'), 'issue': issue_name} + for assignee in resp.get('assignees', []) + ] + if (i + 1) % 10 == 0: + frappe.db.commit() except Exception as e: frappe.log_error(f"Error creating issue: {str(e)}", "Bulk Create Issues") - + if issue_rows: + frappe.get_doc({'doctype': 'Repository Issue', 'repository': repository}).insert(ignore_permissions=True) + _replace_child_table_rows_multi_insert('Repository Issue', repository, 'issues_table', issue_rows) + for issue_name, assignees in issue_assignee_rows.items(): + _replace_child_table_rows_multi_insert('Repository Issue', issue_name, 'assignees_table', assignees) + frappe.db.commit() return {'created': len(created_issues), 'issues': created_issues} @frappe.whitelist() @@ -648,16 +637,17 @@ def create_pull_request(repository, title, head, base, body=None): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - + check_rate_limit(token) payload = {'title': title, 'head': head, 'base': base} - if body: + if body: payload['body'] = body - resp = github_request('POST', f'/repos/{repository}/pulls', token, data=payload) if resp: try: + pr_name = frappe.generate_hash("", 10) doc = frappe.get_doc({ 'doctype': 'Repository Pull Request', + 'name': pr_name, 'repository': repository, 'pr_number': resp.get('number'), 'title': resp.get('title'), @@ -673,83 +663,74 @@ def create_pull_request(repository, title, head, base, body=None): 'updated_at': convert_github_datetime(resp.get('updated_at')) }) doc.insert(ignore_permissions=True) + rows = [ + {'user': frappe.db.get_value("User", {"github_username": reviewer.get('login')}, "name") or reviewer.get('login'), 'pull_request': pr_name} + for reviewer in resp.get('requested_reviewers', []) + ] + _replace_child_table_rows_multi_insert('Repository Pull Request', pr_name, 'reviewers_table', rows) + frappe.db.commit() return {'pull_request': resp, 'local_doc': doc.name} - except Exception: - pass + except Exception as e: + frappe.db.rollback() + frappe.log_error(f"Error creating local PR: {str(e)}", "GitHub Create PR") return resp @frappe.whitelist() def sync_repo_members(repo_full_name): - if not _can_sync_repo(repo_full_name): - frappe.throw(_('You do not have permission to sync this repository.')) - settings = frappe.get_single('GitHub Settings') - token = settings.get_password('personal_access_token') - if not token: - frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - - try: - members = list_repo_members(repo_full_name) - except Exception as e: - frappe.throw(str(e)) - - # Update repository members table try: + if not _can_sync_repo(repo_full_name): + frappe.throw(_('You do not have permission to sync this repository.')) + settings = frappe.get_single('GitHub Settings') + token = settings.get_password('personal_access_token') + if not token: + frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) + check_rate_limit(token) + members = fetch_paginated_data(f'/repos/{repo_full_name}/collaborators', token) repo_doc = frappe.get_doc('Repository', {'full_name': repo_full_name}) - repo_doc.set('members_table', []) + member_rows = [] for m in members or []: - user_info = github_request('GET', f"/users/{m.get('login')}", token) - m_email = user_info.get("email") or "" - repo_doc.append('members_table', { + user_info = github_request('GET', f"/users/{m.get('login')}", token) or {} + member_rows.append({ 'repo_full_name': repo_full_name, 'github_username': m.get('login'), 'github_id': str(m.get('id', '')), 'role': 'maintainer' if m.get('permissions', {}).get('admin') else 'member', - 'email': m_email or '' + 'email': user_info.get('email') or '' }) - + _replace_child_table_rows_multi_insert('Repository', repo_doc.name, 'members_table', member_rows) repo_doc.save(ignore_permissions=True) - except Exception: - pass - - # Update linked projects - projects = frappe.get_all('Project', filters={'repository': repo_full_name}, fields=['name']) - for p in projects: - try: - proj = frappe.get_doc('Project', p.get('name')) - proj.set('project_users', []) - - for m in members or []: - user_info = github_request('GET', f"/users/{m.get('login')}", token) - m_email = user_info.get("email") or "" - username = m.get('login') - erp_user = None - - # Try to find matching ERP user - try: - user_name = frappe.db.get_value("User", {"github_username": username}, "name") - erp_user = user_name - except Exception: - # Try to find by email if available - if m_email: - try: - user_name = frappe.db.get_value("User", {"email": m_email}, "name") - if user_name: - user_doc = frappe.get_doc("User", user_name) - user_doc.github_username = username - user_doc.save(ignore_permissions=True) - except Exception: - pass - - proj.append('project_users', { - 'user': erp_user or username, - 'role': 'Project User' - }) - - proj.save(ignore_permissions=True) - except Exception as e: - frappe.log_error(f"Error updating project {p.get('name')}: {str(e)}") - - return {'members': len(members or [])} + projects = frappe.get_all('Project', filters={'repository': repo_full_name}, fields=['name']) + for i, p in enumerate(projects): + try: + proj = frappe.get_doc('Project', p.get('name')) + project_user_rows = [] + for m in members or []: + user_info = github_request('GET', f"/users/{m.get('login')}", token) or {} + m_email = user_info.get('email') or '' + username = m.get('login') + erp_user = frappe.db.get_value("User", {"github_username": username}, "name") + if not erp_user and m_email: + erp_user = frappe.db.get_value("User", {"email": m_email}, "name") + if erp_user: + user_doc = frappe.get_doc("User", erp_user) + user_doc.github_username = username + user_doc.save(ignore_permissions=True) + project_user_rows.append({ + 'user': erp_user or username, + 'role': 'Project User' + }) + _replace_child_table_rows_multi_insert('Project', proj.name, 'project_users', project_user_rows) + proj.save(ignore_permissions=True) + if (i + 1) % 5 == 0: + frappe.db.commit() + except Exception as e: + frappe.log_error(f"Error updating project {p.get('name')}: {str(e)}", "GitHub Sync Project") + frappe.db.commit() + return {'members': len(members or [])} + except Exception as e: + frappe.db.rollback() + frappe.log_error(f"Error syncing repository members: {str(e)}", "GitHub Sync Members") + raise e @frappe.whitelist() def manage_repo_access(repo_full_name, action, identifier, permission='push'): @@ -758,127 +739,123 @@ def manage_repo_access(repo_full_name, action, identifier, permission='push'): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - + check_rate_limit(token) parts = repo_full_name.split('/') if len(parts) != 2: frappe.throw(_('repo_full_name must be in owner/repo format')) owner, repo = parts - try: if action == 'add_collaborator': - resp = github_request('PUT', f"/repos/{owner}/{repo}/collaborators/{identifier}", token, - data={'permission': permission}) - return resp + resp = github_request('PUT', f"/repos/{owner}/{repo}/collaborators/{identifier}", token, data={'permission': permission}) elif action == 'remove_collaborator': resp = github_request('DELETE', f"/repos/{owner}/{repo}/collaborators/{identifier}", token) - return resp elif action == 'add_team': - resp = github_request('PUT', f"/orgs/{owner}/teams/{identifier}/repos/{owner}/{repo}", token, - data={'permission': permission}) - return resp + resp = github_request('PUT', f"/orgs/{owner}/teams/{identifier}/repos/{owner}/{repo}", token, data={'permission': permission}) elif action == 'remove_team': resp = github_request('DELETE', f"/orgs/{owner}/teams/{identifier}/repos/{owner}/{repo}", token) - return resp else: frappe.throw(_('Unknown action: {0}').format(action)) + return resp except Exception as e: frappe.throw(_('manage_repo_access failed: {0}').format(str(e))) @frappe.whitelist() -def sync_all_repositories(): +def start_sync_all_repositories(): + _require_github_admin() + user = frappe.session.user + frappe.enqueue('erpnext_github_integration.github_api.sync_all_repositories', queue='long', timeout=3600, is_async=True, user=user) + return {'status': 'queued'} + +@frappe.whitelist() +def sync_all_repositories(user=None): _require_github_admin() repos = frappe.get_all('Repository', fields=['full_name']) + total = len(repos) results = {'success': 0, 'failed': 0} - - for r in repos: + _publish_progress(data={'progress': 0, 'total': total, 'msg': 'Starting sync_all_repositories'}, user=user) + for i, r in enumerate(repos): + repo_name = r.get('full_name') try: - sync_repo(r.get('full_name')) + start_ts = time.perf_counter() + sync_repo(repo_name, user=user) results['success'] += 1 + dur = time.perf_counter() - start_ts + if (i + 1) % 10 == 0 or i == len(repos) - 1: + _publish_progress(data={ + 'progress': i + 1, + 'total': total, + 'repo': repo_name, + 'status': 'ok', + 'time_s': round(dur, 2), + 'success': results['success'], + 'failed': results['failed'], + 'msg': f'Synced {repo_name} ({i + 1}/{total})' + }, user=user) except Exception as e: results['failed'] += 1 - frappe.log_error(message=str(e), title=f'GitHub Sync Error - {r.get("full_name")}') - + frappe.db.rollback() + frappe.log_error(f"Error syncing {repo_name}: {str(e)}", f'GitHub Sync Error - {repo_name}') + _publish_progress(data={ + 'progress': i + 1, + 'total': total, + 'repo': repo_name, + 'status': 'error', + 'success': results['success'], + 'failed': results['failed'], + 'msg': f'Failed {repo_name}: {str(e)[:200]}' + }, user=user) + if i < len(repos) - 1: + time.sleep(1) settings = frappe.get_single("GitHub Settings") settings.last_sync = frappe.utils.now() settings.save(ignore_permissions=True) - + frappe.db.commit() + _publish_progress(data={'progress': total, 'total': total, 'msg': 'Sync finished', 'success': results['success'], 'failed': results['failed']}, user=user) return results @frappe.whitelist() def get_repository_activity(repository, days=30): - """Get recent activity for a repository""" try: settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') - - # Validate and convert days + check_rate_limit(token) try: days_int = int(days) except (ValueError, TypeError): - days_int = 30 # Default fallback - - # Ensure days is within reasonable bounds - days_int = max(1, min(days_int, 365)) # Between 1 and 365 days - - # Calculate since date + days_int = 30 + days_int = max(1, min(days_int, 365)) since = (datetime.now() - timedelta(days=days_int)).isoformat() - - # Get activity data - commits = github_request( - 'GET', - f'/repos/{repository}/commits', - token, - params={'since': since, 'per_page': 50} - ) or [] - - issues = github_request( - 'GET', - f'/repos/{repository}/issues', - token, - params={'since': since, 'state': 'all', 'per_page': 20} - ) or [] - - pulls = github_request( - 'GET', - f'/repos/{repository}/pulls', - token, - params={'since': since, 'state': 'all', 'per_page': 20} - ) or [] - - # Filter out pull requests from issues (GitHub API returns PRs in issues) + commits = fetch_paginated_data(f'/repos/{repository}/commits', token, params={'since': since, 'per_page': 50}) + issues = fetch_paginated_data(f'/repos/{repository}/issues', token, params={'since': since, 'state': 'all', 'per_page': 20}) + pulls = fetch_paginated_data(f'/repos/{repository}/pulls', token, params={'since': since, 'state': 'all', 'per_page': 20}) actual_issues = [issue for issue in issues if 'pull_request' not in issue] - return { 'commits': len(commits), 'issues': len(actual_issues), 'pulls': len(pulls), 'period_days': days_int, 'details': { - 'commits': commits[:10], # Return first 10 for preview + 'commits': commits[:10], 'issues': actual_issues[:10], 'pulls': pulls[:10] } } - except Exception as e: - frappe.logger().error(f"Error getting repository activity: {str(e)}") + frappe.log_error(f"Error getting repository activity: {str(e)}", "GitHub Activity") return {'error': str(e)} @frappe.whitelist() def create_repository_webhook(repo_full_name, webhook_url=None, events=None): - """Create a webhook for the repository""" _require_github_admin() settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - + check_rate_limit(token) if not webhook_url: webhook_url = frappe.utils.get_url('/api/method/erpnext_github_integration.webhooks.github_webhook') - if not events: events = ['push', 'pull_request', 'issues', 'issue_comment'] - payload = { 'name': 'web', 'active': True, @@ -888,10 +865,8 @@ def create_repository_webhook(repo_full_name, webhook_url=None, events=None): 'content_type': 'json' } } - if settings.get_password('webhook_secret'): payload['config']['secret'] = settings.get_password('webhook_secret') - try: resp = github_request('POST', f"/repos/{repo_full_name}/hooks", token, data=payload) return resp @@ -900,14 +875,12 @@ def create_repository_webhook(repo_full_name, webhook_url=None, events=None): @frappe.whitelist() def list_repository_webhooks(repo_full_name): - """List all webhooks for a repository""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - + check_rate_limit(token) try: - webhooks = github_request('GET', f"/repos/{repo_full_name}/hooks", token) - return webhooks or [] + return fetch_paginated_data(f"/repos/{repo_full_name}/hooks", token) except Exception as e: frappe.throw(_('Failed to list webhooks: {0}').format(str(e))) \ No newline at end of file diff --git a/erpnext_github_integration/public/js/task_client.js b/erpnext_github_integration/public/js/task_client.js index 76eccc8..c310969 100644 --- a/erpnext_github_integration/public/js/task_client.js +++ b/erpnext_github_integration/public/js/task_client.js @@ -17,9 +17,7 @@ frappe.ui.form.on('Task', { callback: function(r) { if (r.message) { let issue = r.message.issue; - let url = issue.html_url || issue.url; - - frappe.msgprint(__('Created issue: {0}').format(url)); + frappe.msgprint(__('Issue Successfully Created')); // Save both: local doc link & GitHub issue number frm.set_value('github_issue_doc', r.message.local_doc); @@ -49,8 +47,7 @@ frappe.ui.form.on('Task', { callback: function(r) { if (r.message) { let pr = r.message.pull_request; - let url = pr.html_url || pr.url; - frappe.msgprint(__('Created pull request: {0}').format(url)); + frappe.msgprint(__('Pull Request Created Successfully')); frm.set_value('github_pr_number', pr.number); frm.save(); } @@ -96,7 +93,7 @@ frappe.ui.form.on('Task', { assignees: values.assignees }, callback: function(r) { - frappe.msgprint(__('Assigned issue {0}', [issue_no])); + frappe.msgprint(__('Issue Assigned Successfully')); } }); }, __('Assign Issue')); @@ -145,7 +142,7 @@ frappe.ui.form.on('Task', { callback: function(r) { if (r.message) { frappe.msgprint("Message: " + r.message); - frappe.msgprint(__('Reviewers assigned to PR {0}', [pr_number])); + frappe.msgprint(__('Reviewers assigned to PR Successfully')); } } }); From 403b2b84486b7b13a2d6bf230250b760243cb75b Mon Sep 17 00:00:00 2001 From: Yankyyyy Date: Wed, 17 Sep 2025 10:25:17 +0000 Subject: [PATCH 2/3] feat: optimized sync_all function and improved UI --- .../github_settings/github_settings.js | 44 +- erpnext_github_integration/github_api.py | 1035 ++++++++++------- erpnext_github_integration/hooks.py | 10 +- .../public/js/project_client.js | 29 +- 4 files changed, 654 insertions(+), 464 deletions(-) diff --git a/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js b/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js index 27be1e8..54ad64c 100644 --- a/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js +++ b/erpnext_github_integration/erpnext_github_integration/doctype/github_settings/github_settings.js @@ -280,22 +280,42 @@ frappe.ui.form.on("GitHub Settings", { ], primary_action_label: __('Create Issues'), primary_action: function(values) { + let issues; try { - let issues = JSON.parse(values.issues_data); - frappe.call({ - method: 'erpnext_github_integration.github_api.bulk_create_issues', - args: { - repository: values.repository, - issues: issues - }, - callback: function(r) { - frappe.msgprint(__('Issues created successfully')); - d.hide(); - } - }); + issues = JSON.parse(values.issues_data); } catch (e) { frappe.msgprint(__('Invalid JSON format')); + return; } + + const btn = this.get_primary_btn(); + btn.prop('disabled', true).text(__('Creating...')); + + frappe.call({ + method: 'erpnext_github_integration.github_api.bulk_create_issues', + args: { + repository: values.repository, + issues: issues + }, + callback: function(r) { + btn.prop('disabled', false).text(__('Create Issues')); + if (!r || !r.message) { + frappe.msgprint(__('No response from server')); + return; + } + const resp = r.message; + if (resp.created !== undefined) { + frappe.msgprint(__(`Created ${resp.created} issues`)); + } else { + frappe.msgprint(__('Issues created successfully')); + } + d.hide(); + }, + error: function(err) { + btn.prop('disabled', false).text(__('Create Issues')); + frappe.msgprint(__('Error creating issues: ') + (err && err.message || JSON.stringify(err))); + } + }); } }); d.show(); diff --git a/erpnext_github_integration/github_api.py b/erpnext_github_integration/github_api.py index fb7336d..e879f44 100644 --- a/erpnext_github_integration/github_api.py +++ b/erpnext_github_integration/github_api.py @@ -1,41 +1,74 @@ import frappe, json from frappe import _ -from frappe.model.meta import get_table_columns import datetime -import time from datetime import datetime, timedelta from dateutil import parser import pytz -from concurrent.futures import ThreadPoolExecutor -import requests from .github_client import github_request +from frappe.desk.form.assign_to import add, clear +import time def has_role(role): """Compatibility function for different Frappe versions""" try: + # For older Frappe versions return frappe.has_role(role) except AttributeError: + # For Frappe v15+ return role in frappe.get_roles() - + +# Helper function to convert GitHub datetime to MySQL format def convert_github_datetime(dt_string): if not dt_string: return None try: + # Parse ISO 8601 format dt = parser.parse(dt_string) + + # Ensure it's treated as UTC if no timezone info if dt.tzinfo is None: dt = pytz.utc.localize(dt) + elif dt.tzinfo.utcoffset(dt) == pytz.utc.utcoffset(dt): + pass # Already UTC + + # Convert to IST ist_tz = pytz.timezone('Asia/Kolkata') local_dt = dt.astimezone(ist_tz) + + # Return without timezone info for MySQL return local_dt.replace(tzinfo=None).strftime('%Y-%m-%d %H:%M:%S') + except (ValueError, TypeError) as e: frappe.log_error(f'Error parsing datetime {dt_string}: {str(e)}', 'DateTime Parse Error') return None +# Helper function to convert MySQL (IST) datetime to GitHub UTC ISO +def convert_to_github_datetime(local_dt_str): + if not local_dt_str: + return None + try: + dt = datetime.strptime(local_dt_str, '%Y-%m-%d %H:%M:%S') + ist_tz = pytz.timezone('Asia/Kolkata') + local_dt = ist_tz.localize(dt) + utc_dt = local_dt.astimezone(pytz.utc) + return utc_dt.isoformat() + except Exception as e: + frappe.log_error(f'Error converting datetime {local_dt_str}: {str(e)}', 'DateTime Convert Error') + return None + +# Usage +# if not has_role('GitHub Admin'): +# frappe.throw("Permission required") + def _require_github_admin(): if not has_role('GitHub Admin'): frappe.throw(_('Only users with the GitHub Admin role can perform this action.')) def _can_sync_repo(repo_full_name): + """Return True if current user can sync the repo: + - GitHub Admins can always sync + - Project Manager of any Project linked to this repo can sync + """ if has_role('GitHub Admin'): return True projects = frappe.get_all('Project', filters={'repository': repo_full_name}, fields=['name', 'project_manager']) @@ -45,150 +78,79 @@ def _can_sync_repo(repo_full_name): return True return False -def _publish_progress(event='github_sync_progress', data=None, user=None): - try: - frappe.publish_realtime(event, data or {}, user=user) - except Exception: - frappe.log_error(frappe.get_traceback(), "Failed to publish realtime progress") - -def _replace_child_table_rows_multi_insert(parent_doctype, parent_name, child_table_fieldname, rows): - """ - Replace child table rows with a single multi-row INSERT (fast, DB-level). - Ensure docstatus is included and set to 0 for all child table rows. - """ - meta = frappe.get_meta(parent_doctype) - child_meta = frappe.get_meta(meta.get_field(child_table_fieldname).options) - valid_columns = child_meta.get_valid_columns() - base_columns = ["name", "owner", "creation", "modified", "modified_by", - "parent", "parentfield", "parenttype", "idx", "docstatus"] - insert_columns = base_columns + [c for c in valid_columns if c not in base_columns] - - # Delete existing rows - frappe.db.sql(f""" - DELETE FROM `tab{child_meta.name}` - WHERE parent=%s AND parentfield=%s AND parenttype=%s - """, (parent_name, child_table_fieldname, parent_doctype)) - - if not rows: - return - - now = frappe.utils.now_datetime() - values = [] - for idx, row in enumerate(rows, start=1): - base_vals = { - "name": frappe.generate_hash("", 10), - "owner": frappe.session.user, - "creation": now, - "modified": now, - "modified_by": frappe.session.user, - "parent": parent_name, - "parentfield": child_table_fieldname, - "parenttype": parent_doctype, - "idx": idx, - "docstatus": 0 # Explicitly set docstatus to 0 - } - vals = [] - for col in insert_columns: - vals.append(base_vals.get(col, row.get(col))) - values.append(vals) - - placeholders = "(" + ", ".join(["%s"] * len(insert_columns)) + ")" - insert_sql = f""" - INSERT INTO `tab{child_meta.name}` ({", ".join(f"`{c}`" for c in insert_columns)}) - VALUES {", ".join([placeholders] * len(values))} - """ - flat_values = [val for row_vals in values for val in row_vals] - frappe.db.sql(insert_sql, tuple(flat_values)) - -def fetch_paginated_data(path, token, params=None): - params = params or {} - params['per_page'] = 100 - results = [] - page = 1 - while True: - params['page'] = page - data = github_request('GET', path, token, params=params) or [] - results.extend(data) - if len(data) < 100: - break - page += 1 - return results - -def check_rate_limit(token): - response = github_request('GET', '/rate_limit', token) - remaining = response.get('rate', {}).get('remaining', 0) - reset_time = response.get('rate', {}).get('reset', 0) - if remaining < 100: - reset_dt = datetime.fromtimestamp(reset_time) - frappe.throw(f"GitHub API rate limit too low ({remaining} requests remaining). Try again after {reset_dt}.") - return remaining - -def github_request(method, path, token, params=None, data=None): - headers = {'Authorization': f'token {token}', 'Accept': 'application/vnd.github.v3+json'} - url = f"https://api.github.com{path}" - for attempt in range(3): - try: - response = requests.request(method, url, headers=headers, params=params, json=data) - if response.status_code == 429: - retry_after = int(response.headers.get('Retry-After', 60)) - time.sleep(retry_after) - continue - response.raise_for_status() - return response.json() if response.content else {} - except requests.exceptions.RequestException as e: - if attempt == 2: - raise Exception(f"GitHub API request failed: {str(e)}") - time.sleep(2 ** attempt) - return {} - @frappe.whitelist() def test_connection(): + """Test GitHub API connection""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: return {'success': False, 'error': 'GitHub Personal Access Token not configured'} + try: - check_rate_limit(token) user_info = github_request('GET', '/user', token) if user_info and user_info.get('login'): return {'success': True, 'user': user_info.get('login')} - return {'success': False, 'error': 'Invalid response from GitHub API'} + else: + return {'success': False, 'error': 'Invalid response from GitHub API'} except Exception as e: return {'success': False, 'error': str(e)} - + @frappe.whitelist() def get_github_username_by_email(email): + """Fetch GitHub username from GitHub API using email""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') + if not token: return {'success': False, 'error': 'GitHub Personal Access Token not configured'} + try: - check_rate_limit(token) + # Use github_request to search for users by email search_results = github_request('GET', f'/search/users?q={email}+in:email', token) + if search_results and search_results.get('total_count', 0) > 0 and search_results.get('items'): + # Return the first matching username return { 'success': True, 'github_username': search_results['items'][0]['login'], 'total_results': search_results['total_count'] } - return {'success': False, 'error': f'No GitHub user found with email: {email}'} + else: + return { + 'success': False, + 'error': f'No GitHub user found with email: {email}' + } + except Exception as e: return {'success': False, 'error': str(e)} - + @frappe.whitelist() def fetch_all_repositories(organization=None): + """Fetch all repositories from GitHub and create/update them in ERPNext""" try: settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') + if not token: frappe.throw('GitHub Personal Access Token not configured') - check_rate_limit(token) - repos = fetch_paginated_data(f'/orgs/{organization}/repos' if organization else '/user/repos', token) + + # Get repositories from GitHub + if organization: + repos = github_request('GET', f'/orgs/{organization}/repos', token, params={'per_page': 100}) + else: + repos = github_request('GET', '/user/repos', token, params={'per_page': 100, 'affiliation': 'owner'}) + if not repos: return {'success': False, 'message': 'No repositories found'} - created_count = updated_count = 0 + + created_count = 0 + updated_count = 0 + for repo in repos: + # Check if repository already exists repo_name = repo.get('full_name') + existing_repo = frappe.db.exists('Repository', {'full_name': repo_name}) + repo_data = { 'full_name': repo_name, 'repo_name': repo.get('name'), @@ -199,36 +161,44 @@ def fetch_all_repositories(organization=None): 'default_branch': repo.get('default_branch'), 'is_synced': 0 } - if frappe.db.exists('Repository', {'full_name': repo_name}): - doc = frappe.get_doc('Repository', {'full_name': repo_name}) + + if existing_repo: + # Update existing repository + doc = frappe.get_doc('Repository', existing_repo) doc.update(repo_data) - doc.save(ignore_permissions=True) + doc.save() updated_count += 1 + frappe.logger().info(f"Updated repository: {repo_name}") else: - doc = frappe.get_doc({'doctype': 'Repository', **repo_data}) - doc.insert(ignore_permissions=True) + # Create new repository + doc = frappe.get_doc({ + 'doctype': 'Repository', + **repo_data + }) + doc.insert() created_count += 1 - if (created_count + updated_count) % 10 == 0: - frappe.db.commit() - frappe.db.commit() + frappe.logger().info(f"Created repository: {repo_name}") + return { 'success': True, 'message': f'Successfully fetched {len(repos)} repositories. Created: {created_count}, Updated: {updated_count}' } + except Exception as e: - frappe.db.rollback() - frappe.log_error(f"Error fetching repositories: {str(e)}", "GitHub Fetch Repos") + frappe.logger().error(f"Error fetching repositories: {str(e)}") return {'success': False, 'message': f'Error: {str(e)}'} @frappe.whitelist() def get_sync_statistics(): - return { + """Get synchronization statistics""" + stats = { 'repositories': frappe.db.count('Repository'), 'issues': frappe.db.count('Repository Issue'), 'pull_requests': frappe.db.count('Repository Pull Request'), 'members': frappe.db.count('Repository Member'), 'branches': frappe.db.count('Repository Branch') } + return stats @frappe.whitelist() def can_user_sync_repo(repo_full_name): @@ -240,9 +210,11 @@ def list_repositories(organization=None): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) - path = f"/orgs/{organization}/repos" if organization else "/user/repos" - return fetch_paginated_data(path, token) + if organization: + path = f"/orgs/{organization}/repos" + else: + path = "/user/repos" + return github_request('GET', path, token, params={'per_page': 100}) @frappe.whitelist() def list_branches(repo_full_name, per_page=100): @@ -250,8 +222,8 @@ def list_branches(repo_full_name, per_page=100): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) - return fetch_paginated_data(f"/repos/{repo_full_name}/branches", token, params={'per_page': per_page}) + path = f"/repos/{repo_full_name}/branches" + return github_request('GET', path, token, params={'per_page': per_page}) @frappe.whitelist() def list_teams(org_name, per_page=100): @@ -259,8 +231,8 @@ def list_teams(org_name, per_page=100): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) - return fetch_paginated_data(f"/orgs/{org_name}/teams", token, params={'per_page': per_page}) + path = f"/orgs/{org_name}/teams" + return github_request('GET', path, token, params={'per_page': per_page}) @frappe.whitelist() def list_repo_members(repo_full_name, per_page=100): @@ -268,11 +240,11 @@ def list_repo_members(repo_full_name, per_page=100): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) try: - return fetch_paginated_data(f"/repos/{repo_full_name}/collaborators", token, params={'per_page': per_page}) + members = github_request('GET', f"/repos/{repo_full_name}/collaborators", token, params={'per_page': per_page}) except Exception as e: frappe.throw(_('GitHub list repo members failed: {0}').format(str(e))) + return members @frappe.whitelist() def assign_issue(repo_full_name, issue_number, assignees): @@ -280,203 +252,337 @@ def assign_issue(repo_full_name, issue_number, assignees): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + + # normalize assignees input if isinstance(assignees, str): try: assignees = json.loads(assignees) except Exception: assignees = [a.strip() for a in assignees.split(',') if a.strip()] + github_usernames = [] - rows = [] + try: - local = frappe.get_doc('Repository Issue', {'repository': repo_full_name, 'issue_number': int(issue_number)}) + local = frappe.get_doc('Repository Issue', { + 'repository': repo_full_name, + 'issue_number': int(issue_number) + }) + + # Find linked Task (if any) + task = frappe.db.get_value( + 'Task', + { + 'github_repo': repo_full_name, + 'github_issue_number': int(issue_number) + }, + ['name', 'subject'], + as_dict=1 + ) + + # Reset assignees table + local.set('assignees_table', []) + + # Clear existing assignments + clear("Repository Issue", local.name) + if task: + clear("Task", task.name) + for user_id in assignees: + # map ERPNext user → GitHub username gh_username = frappe.db.get_value("User", user_id, "github_username") + if not gh_username: - frappe.log_error(f"User {user_id} has no GitHub username set", "GitHub Assignee Mapping") - continue + frappe.log_error( + f"User {user_id} has no GitHub username set", + "GitHub Assignee Mapping" + ) + continue # skip if no GitHub username + github_usernames.append(gh_username) - rows.append({'user': user_id, 'issue': local.name}) - _replace_child_table_rows_multi_insert('Repository Issue', local.name, 'assignees_table', rows) + + # store ERPNext user (email / id) in local doc + local.append('assignees_table', { + 'issue': local.name, + 'user': user_id + }) + + # also create ERPNext assignments + try: + if task: + add({ + "assign_to": [user_id], + "doctype": "Task", + "name": task.name, + "description": task.subject + }) + add({ + "assign_to": [user_id], + "doctype": "Repository Issue", + "name": local.name, + "description": _("Assigned from GitHub Issue #{0}".format(issue_number)) + }) + except Exception: + frappe.log_error(frappe.get_traceback(), "Failed to create Frappe assignment") + + local.save(ignore_permissions=True) + except Exception: frappe.log_error(frappe.get_traceback(), "Failed to update local Repository Issue assignees") + + # send to GitHub (GitHub API needs GitHub usernames) payload = {'assignees': github_usernames} try: - resp = github_request('PATCH', f"/repos/{repo_full_name}/issues/{issue_number}", token, data=payload) - return resp + resp = github_request( + 'PATCH', + f"/repos/{repo_full_name}/issues/{issue_number}", + token, + data=payload + ) except Exception as e: frappe.throw(_('GitHub assign issue failed: {0}').format(str(e))) + return resp + @frappe.whitelist() def add_pr_reviewer(repo_full_name, pr_number, reviewers): settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) if isinstance(reviewers, str): try: reviewers = json.loads(reviewers) except Exception: reviewers = [r.strip() for r in reviewers.split(',') if r.strip()] - github_usernames = [] - rows = [] - try: - local = frappe.get_doc('Repository Pull Request', {'repository': repo_full_name, 'pr_number': int(pr_number)}) - for user_id in reviewers: - gh_username = frappe.db.get_value("User", user_id, "github_username") - if not gh_username: - frappe.log_error(f"User {user_id} has no GitHub username set", "GitHub Reviewer Mapping") - continue - github_usernames.append(gh_username) - rows.append({'user': user_id, 'pull_request': local.name}) - _replace_child_table_rows_multi_insert('Repository Pull Request', local.name, 'reviewers_table', rows) - except Exception: - frappe.log_error(frappe.get_traceback(), "Failed to update local Repository Pull Request reviewers") - payload = {'reviewers': github_usernames} + payload = {'reviewers': reviewers} try: resp = github_request('POST', f"/repos/{repo_full_name}/pulls/{pr_number}/requested_reviewers", token, data=payload) - return resp except Exception as e: frappe.throw(_('GitHub add PR reviewer failed: {0}').format(str(e))) + if resp: + try: + local = frappe.get_doc('Repository Pull Request', {'repository': repo_full_name, 'pr_number': int(pr_number)}) + # Update reviewers table + local.set('reviewers_table', []) + for reviewer in reviewers: + local.append('reviewers_table', { + 'user': reviewer + }) + local.save(ignore_permissions=True) + except Exception: + pass + return resp + return resp @frappe.whitelist() -def sync_repo(repository, user=None): +def sync_repo(repository): repo_full = repository - _publish_progress(data={'repo': repo_full, 'phase': 'start', 'msg': f'Starting sync for {repo_full}'}, user=user) if not _can_sync_repo(repo_full): frappe.throw(_('You do not have permission to sync this repository.')) settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) - try: - # Initialize or fetch repository document - repo_doc = frappe.get_doc('Repository', {'full_name': repo_full}) if frappe.db.exists('Repository', {'full_name': repo_full}) else frappe.get_doc({ + + # Get repository info + repo_info = github_request('GET', f'/repos/{repo_full}', token) or {} + branches = github_request('GET', f'/repos/{repo_full}/branches', token) or [] + members = github_request('GET', f'/repos/{repo_full}/collaborators', token) or [] + + # Upsert repo doc (without last_synced yet) + existing = frappe.db.exists('Repository', {'full_name': repo_full}) + if existing: + repo_doc = frappe.get_doc('Repository', existing) + is_new = False + else: + repo_doc = frappe.get_doc({ 'doctype': 'Repository', 'full_name': repo_full, 'repo_name': repo_full.split('/')[-1], 'repo_owner': repo_full.split('/')[0], - 'url': f'https://github.com/{repo_full}' - }) - last_synced = repo_doc.last_synced - params = {'state': 'all', 'per_page': 100} - if last_synced: - params['since'] = last_synced.isoformat() - - # Parallel API calls - def fetch_repo_info(): - return github_request('GET', f'/repos/{repo_full}', token) or {} - def fetch_branches(): - return fetch_paginated_data(f'/repos/{repo_full}/branches', token) - def fetch_issues(): - return fetch_paginated_data(f'/repos/{repo_full}/issues', token, params=params) - def fetch_pulls(): - return fetch_paginated_data(f'/repos/{repo_full}/pulls', token, params=params) - def fetch_members(): - return fetch_paginated_data(f'/repos/{repo_full}/collaborators', token) - - with ThreadPoolExecutor(max_workers=4) as executor: - futures = { - 'repo_info': executor.submit(fetch_repo_info), - 'branches': executor.submit(fetch_branches), - 'issues': executor.submit(fetch_issues), - 'pulls': executor.submit(fetch_pulls), - 'members': executor.submit(fetch_members) - } - repo_info = futures['repo_info'].result() - branches = futures['branches'].result() - issues = futures['issues'].result() - pulls = futures['pulls'].result() - members = futures['members'].result() - - # Update repository fields - repo_doc.update({ + 'url': f'https://github.com/{repo_full}', 'github_id': str(repo_info.get('id', '')), 'visibility': 'Private' if repo_info.get('private') else 'Public', 'default_branch': repo_info.get('default_branch', 'main'), - 'is_synced': 1, - 'last_synced': frappe.utils.now() + 'is_synced': 1 }) - - # Prepare branches - branch_rows = [] - for b in branches: - commits = github_request('GET', f'/repos/{repo_full}/commits', token, params={'sha': b.get('name'), 'per_page': 1}) or [] - commit_date = convert_github_datetime(commits[0].get('commit', {}).get('author', {}).get('date')) if commits else None - branch_rows.append({ - 'repo_full_name': repo_full, - 'branch_name': b.get('name'), - 'commit_sha': b.get('commit', {}).get('sha'), - 'protected': b.get('protected', False), - 'last_updated': commit_date or '' - }) - - # Prepare members - member_rows = [] - for m in members: - user_info = github_request('GET', f"/users/{m.get('login')}", token) or {} - member_rows.append({ - 'repo_full_name': repo_full, - 'github_username': m.get('login'), - 'github_id': str(m.get('id', '')), - 'role': 'maintainer' if m.get('permissions', {}).get('admin') else 'member', - 'email': user_info.get('email') or '' - }) - - # Update child tables for Repository - _replace_child_table_rows_multi_insert('Repository', repo_doc.name, 'branches_table', branch_rows) - _replace_child_table_rows_multi_insert('Repository', repo_doc.name, 'members_table', member_rows) + is_new = True + + repo_doc.github_id = str(repo_info.get('id', '')) + repo_doc.visibility = 'Private' if repo_info.get('private') else 'Public' + repo_doc.default_branch = repo_info.get('default_branch', 'main') + + last_synced_local = getattr(repo_doc, 'last_synced', None) if not is_new else None + since_utc = convert_to_github_datetime(last_synced_local) + + params = {'state': 'all'} + if since_utc: + params['since'] = since_utc + + issues = github_request('GET', f'/repos/{repo_full}/issues', token, params=params) or [] + pulls = github_request('GET', f'/repos/{repo_full}/pulls', token, params=params) or [] + + if is_new: + repo_doc.insert(ignore_permissions=True) + else: repo_doc.save(ignore_permissions=True) - - # Handle issues - issue_rows = [] - issue_assignee_rows = {} - for issue in issues: - if issue.get('pull_request'): - continue - issue_name = frappe.generate_hash("", 10) - issue_rows.append({ - 'name': issue_name, + + # Collect all GitHub logins for assignees/reviewers to batch query ERP users + all_github_logins = set() + for issue in issues: + if issue.get('pull_request'): + continue + for a in issue.get('assignees', []): + all_github_logins.add(a.get('login')) + for pr in pulls: + for r in pr.get('requested_reviewers', []): + all_github_logins.add(r.get('login')) + + gh_to_erp = {} + if all_github_logins: + users = frappe.get_all( + 'User', + filters={'github_username': ['in', list(all_github_logins)]}, + fields=['name', 'github_username'] + ) + gh_to_erp = {u['github_username']: u['name'] for u in users} + + # Clear and update branches + repo_doc.set('branches_table', []) + for b in branches: + branch_name = b.get('name') + commit_sha = b.get('commit', {}).get('sha') + + # Get the commit details for last updated date + commit_date = '' + if commit_sha: + commit = github_request('GET', f'/repos/{repo_full}/commits/{commit_sha}', token) + if isinstance(commit, dict): + commit_date_str = commit.get('commit', {}).get('author', {}).get('date', '') + commit_date = convert_github_datetime(commit_date_str) if commit_date_str else '' + else: + print(f"Unexpected commit response for {repo_full}, branch {branch_name}, SHA {commit_sha}: {commit}") + commit_date = '' + + repo_doc.append('branches_table', { + 'repo_full_name': repo_full, + 'branch_name': branch_name, + 'commit_sha': commit_sha or '', + 'protected': b.get('protected', False), + 'last_updated': commit_date + }) + + # Clear and update members + repo_doc.set('members_table', []) + for m in members: + user_info = github_request('GET', f"/users/{m.get('login')}", token) + m_email = user_info.get("email") or "" + repo_doc.append('members_table', { + 'repo_full_name': repo_full, + 'github_username': m.get('login'), + 'github_id': str(m.get('id', '')), + 'role': 'maintainer' if m.get('permissions', {}).get('admin') else 'member', + 'email': m_email or '' + }) + + repo_doc.save(ignore_permissions=True) + + # Sync issues + for issue in issues: + if issue.get('pull_request'): + continue # Skip pull requests (they're handled separately) + + # Check if issue exists + issue_filters = {'repository': repo_full, 'issue_number': issue.get('number')} + existing_issue = frappe.db.exists('Repository Issue', issue_filters) + + assignees_gh = issue.get('assignees', []) + labels_list = [lab.get('name') for lab in issue.get('labels', [])] + + if existing_issue: + # Update existing issue + local = frappe.get_doc('Repository Issue', issue_filters) + local.title = issue.get('title') + local.body = issue.get('body') or '' + local.state = issue.get('state') + local.labels = ','.join(labels_list) + local.url = issue.get('html_url') + local.github_id = str(issue.get('id', '')) + local.updated_at = convert_github_datetime(issue.get('updated_at')) + + # Update assignees + local.set('assignees_table', []) + for assignee in assignees_gh: + erp_user = gh_to_erp.get(assignee.get('login'), assignee.get('login')) + local.append('assignees_table', { + 'user': erp_user, + 'issue': local.name + }) + + local.save(ignore_permissions=True) + else: + # Create new issue + issue_doc = frappe.get_doc({ + 'doctype': 'Repository Issue', 'repository': repo_full, 'issue_number': issue.get('number'), 'title': issue.get('title'), 'body': issue.get('body') or '', 'state': issue.get('state'), - 'labels': ','.join([lab.get('name') for lab in issue.get('labels', [])]), + 'labels': ','.join(labels_list), 'url': issue.get('html_url'), 'github_id': str(issue.get('id', '')), 'created_at': convert_github_datetime(issue.get('created_at')), 'updated_at': convert_github_datetime(issue.get('updated_at')) }) - issue_assignee_rows[issue_name] = [ - {'user': frappe.db.get_value("User", {"github_username": assignee.get('login')}, "name") or assignee.get('login'), 'issue': issue_name} - for assignee in issue.get('assignees', []) - ] - - if issue_rows: - # Delete existing issues to avoid duplicates - frappe.db.sql(f""" - DELETE FROM `tabRepository Issue` - WHERE repository=%s AND issue_number IN %s - """, (repo_full, [row['issue_number'] for row in issue_rows])) - for issue_row in issue_rows: - issue_doc = frappe.get_doc({ - 'doctype': 'Repository Issue', - **issue_row + issue_doc.insert(ignore_permissions=True) + + # Add assignees after insert + for assignee in assignees_gh: + erp_user = gh_to_erp.get(assignee.get('login'), assignee.get('login')) + issue_doc.append('assignees_table', { + 'user': erp_user, + 'issue': issue_doc.name + }) + issue_doc.save(ignore_permissions=True) + + # Sync pull requests + for pr in pulls: + # Check if PR exists + pr_filters = {'repository': repo_full, 'pr_number': pr.get('number')} + existing_pr = frappe.db.exists('Repository Pull Request', pr_filters) + + reviewers_gh = pr.get('requested_reviewers', []) + + if existing_pr: + # Update existing PR + local = frappe.get_doc('Repository Pull Request', pr_filters) + local.title = pr.get('title') + local.body = pr.get('body') or '' + local.state = pr.get('state') + local.head_branch = pr.get('head', {}).get('ref') + local.base_branch = pr.get('base', {}).get('ref') + local.author = pr.get('user', {}).get('login') + local.mergeable_state = pr.get('mergeable_state') + local.github_id = str(pr.get('id', '')) + local.url = pr.get('html_url') + local.updated_at = convert_github_datetime(pr.get('updated_at')) + + # Update reviewers + local.set('reviewers_table', []) + for reviewer in reviewers_gh: + gh_login = reviewer.get('login') + erp_user = gh_to_erp.get(gh_login, gh_login) + local.append('reviewers_table', { + 'user': erp_user, + 'pull_request': local.name }) - issue_doc.insert(ignore_permissions=True) - if issue_name in issue_assignee_rows: - _replace_child_table_rows_multi_insert('Repository Issue', issue_row['name'], 'assignees_table', issue_assignee_rows[issue_row['name']]) - - # Handle pull requests - pr_rows = [] - pr_reviewer_rows = {} - for pr in pulls: - pr_name = frappe.generate_hash("", 10) - pr_rows.append({ - 'name': pr_name, + + local.save(ignore_permissions=True) + else: + # Create new PR + pr_doc = frappe.get_doc({ + 'doctype': 'Repository Pull Request', 'repository': repo_full, 'pr_number': pr.get('number'), 'title': pr.get('title'), @@ -491,44 +597,30 @@ def fetch_members(): 'created_at': convert_github_datetime(pr.get('created_at')), 'updated_at': convert_github_datetime(pr.get('updated_at')) }) - pr_reviewer_rows[pr_name] = [ - {'user': frappe.db.get_value("User", {"github_username": reviewer.get('login')}, "name") or reviewer.get('login'), 'pull_request': pr_name} - for reviewer in pr.get('requested_reviewers', []) - ] - - if pr_rows: - # Delete existing pull requests to avoid duplicates - frappe.db.sql(f""" - DELETE FROM `tabRepository Pull Request` - WHERE repository=%s AND pr_number IN %s - """, (repo_full, [row['pr_number'] for row in pr_rows])) - for pr_row in pr_rows: - pr_doc = frappe.get_doc({ - 'doctype': 'Repository Pull Request', - **pr_row + pr_doc.insert(ignore_permissions=True) + + # Add reviewers after insert + for reviewer in reviewers_gh: + gh_login = reviewer.get('login') + erp_user = gh_to_erp.get(gh_login, gh_login) + pr_doc.append('reviewers_table', { + 'user': erp_user, + 'pull_request': pr_doc.name }) - pr_doc.insert(ignore_permissions=True) - if pr_row['name'] in pr_reviewer_rows: - _replace_child_table_rows_multi_insert('Repository Pull Request', pr_row['name'], 'reviewers_table', pr_reviewer_rows[pr_row['name']]) - - frappe.db.commit() - - if len(issues) >= 100 or len(pulls) >= 100: - _publish_progress(data={'repo': repo_full, 'phase': 'done', 'msg': f'Sync complete for {repo_full}', 'issues': len([i for i in issues if not i.get('pull_request')]), 'pulls': len(pulls)}, user=user) - - return { - 'success': True, - 'message': f'Synced repository {repo_full}', - 'branches': len(branches), - 'issues': len([i for i in issues if not i.get('pull_request')]), - 'pulls': len(pulls), - 'members': len(members) - } - except Exception as e: - frappe.db.rollback() - frappe.log_error(f"Error syncing repository {repo_full}: {str(e)}", "GitHub Sync Error") - _publish_progress(data={'repo': repo_full, 'phase': 'error', 'msg': str(e)}, user=user) - raise e + pr_doc.save(ignore_permissions=True) + + # Update last_synced at the end + repo_doc.last_synced = frappe.utils.now() + repo_doc.save(ignore_permissions=True) + + return { + 'success': True, + 'message': f'Synced repository {repo_full}', + 'branches': len(branches), + 'issues': len(issues), + 'pulls': len(pulls), + 'members': len(members) + } @frappe.whitelist() def create_issue(repository, title, body=None, assignees=None, labels=None): @@ -536,9 +628,9 @@ def create_issue(repository, title, body=None, assignees=None, labels=None): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + payload = {'title': title} - if body: + if body: payload['body'] = body if assignees: if isinstance(assignees, str): @@ -554,13 +646,12 @@ def create_issue(repository, title, body=None, assignees=None, labels=None): except Exception: labels = [l.strip() for l in labels.split(',') if l.strip()] payload['labels'] = labels + resp = github_request('POST', f'/repos/{repository}/issues', token, data=payload) if resp: try: - issue_name = frappe.generate_hash("", 10) doc = frappe.get_doc({ 'doctype': 'Repository Issue', - 'name': issue_name, 'repository': repository, 'issue_number': resp.get('number'), 'title': resp.get('title'), @@ -573,38 +664,41 @@ def create_issue(repository, title, body=None, assignees=None, labels=None): 'updated_at': convert_github_datetime(resp.get('updated_at')) }) doc.insert(ignore_permissions=True) - rows = [ - {'user': frappe.db.get_value("User", {"github_username": assignee.get('login')}, "name") or assignee.get('login'), 'issue': issue_name} - for assignee in resp.get('assignees', []) - ] - _replace_child_table_rows_multi_insert('Repository Issue', issue_name, 'assignees_table', rows) - frappe.db.commit() + + # Add assignees after insert + for assignee in resp.get('assignees', []): + gh_login = assignee.get('login') + erp_user = frappe.db.get_value("User", {"github_username": gh_login}, "name") or gh_login + doc.append('assignees_table', { + 'user': erp_user, + 'issue': doc.name + }) + doc.save(ignore_permissions=True) return {'issue': resp, 'local_doc': doc.name} - except Exception as e: - frappe.db.rollback() - frappe.log_error(f"Error creating local issue: {str(e)}", "GitHub Create Issue") + except Exception: + pass return resp @frappe.whitelist() def bulk_create_issues(repository, issues): + """Bulk create multiple issues in a repository""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + if isinstance(issues, str): issues = json.loads(issues) + created_issues = [] - issue_rows = [] - issue_assignee_rows = {} - for i, issue_data in enumerate(issues): + for issue_data in issues: try: resp = github_request('POST', f'/repos/{repository}/issues', token, data=issue_data) if resp: created_issues.append(resp) - issue_name = frappe.generate_hash("", 10) - issue_rows.append({ - 'name': issue_name, + # Create local record + doc = frappe.get_doc({ + 'doctype': 'Repository Issue', 'repository': repository, 'issue_number': resp.get('number'), 'title': resp.get('title'), @@ -615,20 +709,10 @@ def bulk_create_issues(repository, issues): 'created_at': convert_github_datetime(resp.get('created_at')), 'updated_at': convert_github_datetime(resp.get('updated_at')) }) - issue_assignee_rows[issue_name] = [ - {'user': frappe.db.get_value("User", {"github_username": assignee.get('login')}, "name") or assignee.get('login'), 'issue': issue_name} - for assignee in resp.get('assignees', []) - ] - if (i + 1) % 10 == 0: - frappe.db.commit() + doc.insert(ignore_permissions=True) except Exception as e: frappe.log_error(f"Error creating issue: {str(e)}", "Bulk Create Issues") - if issue_rows: - frappe.get_doc({'doctype': 'Repository Issue', 'repository': repository}).insert(ignore_permissions=True) - _replace_child_table_rows_multi_insert('Repository Issue', repository, 'issues_table', issue_rows) - for issue_name, assignees in issue_assignee_rows.items(): - _replace_child_table_rows_multi_insert('Repository Issue', issue_name, 'assignees_table', assignees) - frappe.db.commit() + return {'created': len(created_issues), 'issues': created_issues} @frappe.whitelist() @@ -637,17 +721,16 @@ def create_pull_request(repository, title, head, base, body=None): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + payload = {'title': title, 'head': head, 'base': base} - if body: + if body: payload['body'] = body + resp = github_request('POST', f'/repos/{repository}/pulls', token, data=payload) if resp: try: - pr_name = frappe.generate_hash("", 10) doc = frappe.get_doc({ 'doctype': 'Repository Pull Request', - 'name': pr_name, 'repository': repository, 'pr_number': resp.get('number'), 'title': resp.get('title'), @@ -663,74 +746,83 @@ def create_pull_request(repository, title, head, base, body=None): 'updated_at': convert_github_datetime(resp.get('updated_at')) }) doc.insert(ignore_permissions=True) - rows = [ - {'user': frappe.db.get_value("User", {"github_username": reviewer.get('login')}, "name") or reviewer.get('login'), 'pull_request': pr_name} - for reviewer in resp.get('requested_reviewers', []) - ] - _replace_child_table_rows_multi_insert('Repository Pull Request', pr_name, 'reviewers_table', rows) - frappe.db.commit() return {'pull_request': resp, 'local_doc': doc.name} - except Exception as e: - frappe.db.rollback() - frappe.log_error(f"Error creating local PR: {str(e)}", "GitHub Create PR") + except Exception: + pass return resp @frappe.whitelist() def sync_repo_members(repo_full_name): + if not _can_sync_repo(repo_full_name): + frappe.throw(_('You do not have permission to sync this repository.')) + settings = frappe.get_single('GitHub Settings') + token = settings.get_password('personal_access_token') + if not token: + frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) + + try: + members = list_repo_members(repo_full_name) + except Exception as e: + frappe.throw(str(e)) + + # Update repository members table try: - if not _can_sync_repo(repo_full_name): - frappe.throw(_('You do not have permission to sync this repository.')) - settings = frappe.get_single('GitHub Settings') - token = settings.get_password('personal_access_token') - if not token: - frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) - members = fetch_paginated_data(f'/repos/{repo_full_name}/collaborators', token) repo_doc = frappe.get_doc('Repository', {'full_name': repo_full_name}) - member_rows = [] + repo_doc.set('members_table', []) for m in members or []: - user_info = github_request('GET', f"/users/{m.get('login')}", token) or {} - member_rows.append({ + user_info = github_request('GET', f"/users/{m.get('login')}", token) + m_email = user_info.get("email") or "" + repo_doc.append('members_table', { 'repo_full_name': repo_full_name, 'github_username': m.get('login'), 'github_id': str(m.get('id', '')), 'role': 'maintainer' if m.get('permissions', {}).get('admin') else 'member', - 'email': user_info.get('email') or '' + 'email': m_email or '' }) - _replace_child_table_rows_multi_insert('Repository', repo_doc.name, 'members_table', member_rows) + repo_doc.save(ignore_permissions=True) - projects = frappe.get_all('Project', filters={'repository': repo_full_name}, fields=['name']) - for i, p in enumerate(projects): - try: - proj = frappe.get_doc('Project', p.get('name')) - project_user_rows = [] - for m in members or []: - user_info = github_request('GET', f"/users/{m.get('login')}", token) or {} - m_email = user_info.get('email') or '' - username = m.get('login') - erp_user = frappe.db.get_value("User", {"github_username": username}, "name") - if not erp_user and m_email: - erp_user = frappe.db.get_value("User", {"email": m_email}, "name") - if erp_user: - user_doc = frappe.get_doc("User", erp_user) - user_doc.github_username = username - user_doc.save(ignore_permissions=True) - project_user_rows.append({ - 'user': erp_user or username, - 'role': 'Project User' - }) - _replace_child_table_rows_multi_insert('Project', proj.name, 'project_users', project_user_rows) - proj.save(ignore_permissions=True) - if (i + 1) % 5 == 0: - frappe.db.commit() - except Exception as e: - frappe.log_error(f"Error updating project {p.get('name')}: {str(e)}", "GitHub Sync Project") - frappe.db.commit() - return {'members': len(members or [])} - except Exception as e: - frappe.db.rollback() - frappe.log_error(f"Error syncing repository members: {str(e)}", "GitHub Sync Members") - raise e + except Exception: + pass + + # Update linked projects + projects = frappe.get_all('Project', filters={'repository': repo_full_name}, fields=['name']) + for p in projects: + try: + proj = frappe.get_doc('Project', p.get('name')) + proj.set('project_users', []) + + for m in members or []: + user_info = github_request('GET', f"/users/{m.get('login')}", token) + m_email = user_info.get("email") or "" + username = m.get('login') + erp_user = None + + # Try to find matching ERP user + try: + user_name = frappe.db.get_value("User", {"github_username": username}, "name") + erp_user = user_name + except Exception: + # Try to find by email if available + if m_email: + try: + user_name = frappe.db.get_value("User", {"email": m_email}, "name") + if user_name: + user_doc = frappe.get_doc("User", user_name) + user_doc.github_username = username + user_doc.save(ignore_permissions=True) + except Exception: + pass + + proj.append('project_users', { + 'user': erp_user or username, + 'role': 'Project User' + }) + + proj.save(ignore_permissions=True) + except Exception as e: + frappe.log_error(f"Error updating project {p.get('name')}: {str(e)}") + + return {'members': len(members or [])} @frappe.whitelist() def manage_repo_access(repo_full_name, action, identifier, permission='push'): @@ -739,123 +831,174 @@ def manage_repo_access(repo_full_name, action, identifier, permission='push'): token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + parts = repo_full_name.split('/') if len(parts) != 2: frappe.throw(_('repo_full_name must be in owner/repo format')) owner, repo = parts + try: if action == 'add_collaborator': - resp = github_request('PUT', f"/repos/{owner}/{repo}/collaborators/{identifier}", token, data={'permission': permission}) + resp = github_request('PUT', f"/repos/{owner}/{repo}/collaborators/{identifier}", token, + data={'permission': permission}) + return resp elif action == 'remove_collaborator': resp = github_request('DELETE', f"/repos/{owner}/{repo}/collaborators/{identifier}", token) + return resp elif action == 'add_team': - resp = github_request('PUT', f"/orgs/{owner}/teams/{identifier}/repos/{owner}/{repo}", token, data={'permission': permission}) + resp = github_request('PUT', f"/orgs/{owner}/teams/{identifier}/repos/{owner}/{repo}", token, + data={'permission': permission}) + return resp elif action == 'remove_team': resp = github_request('DELETE', f"/orgs/{owner}/teams/{identifier}/repos/{owner}/{repo}", token) + return resp else: frappe.throw(_('Unknown action: {0}').format(action)) - return resp except Exception as e: frappe.throw(_('manage_repo_access failed: {0}').format(str(e))) -@frappe.whitelist() -def start_sync_all_repositories(): - _require_github_admin() - user = frappe.session.user - frappe.enqueue('erpnext_github_integration.github_api.sync_all_repositories', queue='long', timeout=3600, is_async=True, user=user) - return {'status': 'queued'} - -@frappe.whitelist() -def sync_all_repositories(user=None): +def background_sync_all_repositories(): + """Background job for syncing all repositories with progress updates""" _require_github_admin() repos = frappe.get_all('Repository', fields=['full_name']) total = len(repos) + progress = 0 results = {'success': 0, 'failed': 0} - _publish_progress(data={'progress': 0, 'total': total, 'msg': 'Starting sync_all_repositories'}, user=user) - for i, r in enumerate(repos): + start_time = time.time() + + for r in repos: repo_name = r.get('full_name') + frappe.publish_realtime( + event='github_sync_progress', + message={ + 'progress': progress, + 'total': total, + 'repo': repo_name, + 'phase': 'syncing', + 'time_s': round(time.time() - start_time, 1) + } + ) + try: - start_ts = time.perf_counter() - sync_repo(repo_name, user=user) + sync_repo(repo_name) results['success'] += 1 - dur = time.perf_counter() - start_ts - if (i + 1) % 10 == 0 or i == len(repos) - 1: - _publish_progress(data={ - 'progress': i + 1, - 'total': total, - 'repo': repo_name, - 'status': 'ok', - 'time_s': round(dur, 2), - 'success': results['success'], - 'failed': results['failed'], - 'msg': f'Synced {repo_name} ({i + 1}/{total})' - }, user=user) + status = 'success' except Exception as e: results['failed'] += 1 - frappe.db.rollback() - frappe.log_error(f"Error syncing {repo_name}: {str(e)}", f'GitHub Sync Error - {repo_name}') - _publish_progress(data={ - 'progress': i + 1, + status = 'failed' + frappe.log_error(message=str(e), title=f'GitHub Sync Error - {repo_name}') + + progress += 1 + frappe.publish_realtime( + event='github_sync_progress', + message={ + 'progress': progress, 'total': total, 'repo': repo_name, - 'status': 'error', + 'status': status, 'success': results['success'], 'failed': results['failed'], - 'msg': f'Failed {repo_name}: {str(e)[:200]}' - }, user=user) - if i < len(repos) - 1: - time.sleep(1) + 'time_s': round(time.time() - start_time, 1) + } + ) + settings = frappe.get_single("GitHub Settings") settings.last_sync = frappe.utils.now() settings.save(ignore_permissions=True) - frappe.db.commit() - _publish_progress(data={'progress': total, 'total': total, 'msg': 'Sync finished', 'success': results['success'], 'failed': results['failed']}, user=user) - return results + + frappe.publish_realtime( + event='github_sync_progress', + message={ + 'progress': total, + 'total': total, + 'msg': 'completed', + 'success': results['success'], + 'failed': results['failed'], + 'time_s': round(time.time() - start_time, 1) + } + ) + +@frappe.whitelist() +def start_sync_all_repositories(): + """Start background sync of all repositories""" + frappe.enqueue('erpnext_github_integration.github_api.background_sync_all_repositories') + return {'status': 'queued'} @frappe.whitelist() def get_repository_activity(repository, days=30): + """Get recent activity for a repository""" try: settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') - check_rate_limit(token) + + # Validate and convert days try: days_int = int(days) except (ValueError, TypeError): - days_int = 30 - days_int = max(1, min(days_int, 365)) + days_int = 30 # Default fallback + + # Ensure days is within reasonable bounds + days_int = max(1, min(days_int, 365)) # Between 1 and 365 days + + # Calculate since date since = (datetime.now() - timedelta(days=days_int)).isoformat() - commits = fetch_paginated_data(f'/repos/{repository}/commits', token, params={'since': since, 'per_page': 50}) - issues = fetch_paginated_data(f'/repos/{repository}/issues', token, params={'since': since, 'state': 'all', 'per_page': 20}) - pulls = fetch_paginated_data(f'/repos/{repository}/pulls', token, params={'since': since, 'state': 'all', 'per_page': 20}) + + # Get activity data + commits = github_request( + 'GET', + f'/repos/{repository}/commits', + token, + params={'since': since, 'per_page': 50} + ) or [] + + issues = github_request( + 'GET', + f'/repos/{repository}/issues', + token, + params={'since': since, 'state': 'all', 'per_page': 20} + ) or [] + + pulls = github_request( + 'GET', + f'/repos/{repository}/pulls', + token, + params={'since': since, 'state': 'all', 'per_page': 20} + ) or [] + + # Filter out pull requests from issues (GitHub API returns PRs in issues) actual_issues = [issue for issue in issues if 'pull_request' not in issue] + return { 'commits': len(commits), 'issues': len(actual_issues), 'pulls': len(pulls), 'period_days': days_int, 'details': { - 'commits': commits[:10], + 'commits': commits[:10], # Return first 10 for preview 'issues': actual_issues[:10], 'pulls': pulls[:10] } } + except Exception as e: - frappe.log_error(f"Error getting repository activity: {str(e)}", "GitHub Activity") + frappe.logger().error(f"Error getting repository activity: {str(e)}") return {'error': str(e)} @frappe.whitelist() def create_repository_webhook(repo_full_name, webhook_url=None, events=None): + """Create a webhook for the repository""" _require_github_admin() settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + if not webhook_url: webhook_url = frappe.utils.get_url('/api/method/erpnext_github_integration.webhooks.github_webhook') + if not events: events = ['push', 'pull_request', 'issues', 'issue_comment'] + payload = { 'name': 'web', 'active': True, @@ -865,8 +1008,10 @@ def create_repository_webhook(repo_full_name, webhook_url=None, events=None): 'content_type': 'json' } } + if settings.get_password('webhook_secret'): payload['config']['secret'] = settings.get_password('webhook_secret') + try: resp = github_request('POST', f"/repos/{repo_full_name}/hooks", token, data=payload) return resp @@ -875,12 +1020,14 @@ def create_repository_webhook(repo_full_name, webhook_url=None, events=None): @frappe.whitelist() def list_repository_webhooks(repo_full_name): + """List all webhooks for a repository""" settings = frappe.get_single('GitHub Settings') token = settings.get_password('personal_access_token') if not token: frappe.throw(_('GitHub Personal Access Token not configured in GitHub Settings')) - check_rate_limit(token) + try: - return fetch_paginated_data(f"/repos/{repo_full_name}/hooks", token) + webhooks = github_request('GET', f"/repos/{repo_full_name}/hooks", token) + return webhooks or [] except Exception as e: frappe.throw(_('Failed to list webhooks: {0}').format(str(e))) \ No newline at end of file diff --git a/erpnext_github_integration/hooks.py b/erpnext_github_integration/hooks.py index e1be98b..6e8d8d8 100644 --- a/erpnext_github_integration/hooks.py +++ b/erpnext_github_integration/hooks.py @@ -180,11 +180,11 @@ # ], # } -scheduler_events = { - "hourly": [ - "erpnext_github_integration.github_api.sync_all_repositories" - ] -} +# scheduler_events = { +# "hourly": [ +# "erpnext_github_integration.github_api.sync_all_repositories" +# ] +# } # Testing # ------- diff --git a/erpnext_github_integration/public/js/project_client.js b/erpnext_github_integration/public/js/project_client.js index df419b2..efea798 100644 --- a/erpnext_github_integration/public/js/project_client.js +++ b/erpnext_github_integration/public/js/project_client.js @@ -8,13 +8,14 @@ frappe.ui.form.on('Project', { } frappe.call({ method: 'erpnext_github_integration.github_api.sync_repo_members', - args: {repo_full_name: repo}, + args: { repo_full_name: repo }, callback: function(r) { frappe.msgprint(__('Repository members synced to Project users.')); frm.reload_doc(); } }); }); + frm.add_custom_button(__('Sync Repository Data'), function() { let repo = frm.doc.repository; if (!repo) { @@ -23,7 +24,7 @@ frappe.ui.form.on('Project', { } frappe.call({ method: 'erpnext_github_integration.github_api.sync_repo', - args: {repository: repo}, + args: { repository: repo }, callback: function(r) { frappe.msgprint(__('Repository sync completed.')); frm.reload_doc(); @@ -33,5 +34,27 @@ frappe.ui.form.on('Project', { } }); }); + }, + + after_save: function(frm) { + if (frm.doc.repository) { + frappe.call({ + method: "frappe.client.set_value", + args: { + doctype: "Repository", + name: frm.doc.repository, + fieldname: "project", + value: frm.doc.name + }, + callback: function(r) { + if (!r.exc) { + frappe.show_alert({ + message: __("Repository linked with Project"), + indicator: "green" + }); + } + } + }); + } } -}); +}); \ No newline at end of file From 10564b2bea1fc8bae8714c78e7fd75f1bc523610 Mon Sep 17 00:00:00 2001 From: Yankyyyy Date: Wed, 17 Sep 2025 10:45:07 +0000 Subject: [PATCH 3/3] bug: datetime error bug fix --- erpnext_github_integration/github_api.py | 31 ++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/erpnext_github_integration/github_api.py b/erpnext_github_integration/github_api.py index e879f44..e80e10e 100644 --- a/erpnext_github_integration/github_api.py +++ b/erpnext_github_integration/github_api.py @@ -43,17 +43,34 @@ def convert_github_datetime(dt_string): return None # Helper function to convert MySQL (IST) datetime to GitHub UTC ISO -def convert_to_github_datetime(local_dt_str): - if not local_dt_str: +from datetime import datetime +import pytz +import frappe + +# Helper function to convert MySQL (IST) datetime to GitHub UTC ISO +def convert_to_github_datetime(local_dt): + if not local_dt: return None try: - dt = datetime.strptime(local_dt_str, '%Y-%m-%d %H:%M:%S') + # Ensure dt is a datetime object + if isinstance(local_dt, datetime): + dt = local_dt + else: + dt = datetime.strptime(local_dt, '%Y-%m-%d %H:%M:%S') + + # Localize to IST if naive ist_tz = pytz.timezone('Asia/Kolkata') - local_dt = ist_tz.localize(dt) - utc_dt = local_dt.astimezone(pytz.utc) - return utc_dt.isoformat() + if dt.tzinfo is None: + dt = ist_tz.localize(dt) + + # Convert to UTC + utc_dt = dt.astimezone(pytz.utc) + + # Return ISO 8601 with "Z" suffix (GitHub standard) + return utc_dt.replace(microsecond=0).isoformat().replace('+00:00', 'Z') + except Exception as e: - frappe.log_error(f'Error converting datetime {local_dt_str}: {str(e)}', 'DateTime Convert Error') + frappe.log_error(f'Error converting datetime {local_dt}: {str(e)}', 'DateTime Convert Error') return None # Usage