i
This commit is contained in:
parent
48b10b1529
commit
cd7708dfc1
|
@ -22,8 +22,6 @@ class DiscordDataClient(discord.Client):
|
||||||
"""Custom Discord client for collecting user data."""
|
"""Custom Discord client for collecting user data."""
|
||||||
|
|
||||||
def __init__(self, config: Config, database):
|
def __init__(self, config: Config, database):
|
||||||
|
|
||||||
|
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
self.config = config
|
self.config = config
|
||||||
|
@ -37,15 +35,51 @@ class DiscordDataClient(discord.Client):
|
||||||
self.processed_users: Set[int] = set()
|
self.processed_users: Set[int] = set()
|
||||||
self.target_servers = set(config.get_target_servers())
|
self.target_servers = set(config.get_target_servers())
|
||||||
|
|
||||||
# Start background tasks
|
# Initialize tasks properly - don't start them yet
|
||||||
self.cleanup_task.start()
|
self._setup_tasks()
|
||||||
self.stats_task.start()
|
|
||||||
|
def _setup_tasks(self):
|
||||||
|
"""Set up the background tasks."""
|
||||||
|
@tasks.loop(hours=1)
|
||||||
|
async def cleanup_task():
|
||||||
|
"""Periodic cleanup task."""
|
||||||
|
try:
|
||||||
|
# Clean up old backups
|
||||||
|
await self.database.cleanup_old_backups()
|
||||||
|
|
||||||
|
# Clear processed users set to allow re-processing
|
||||||
|
self.processed_users.clear()
|
||||||
|
|
||||||
|
self.logger.info("Cleanup task completed")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.error(f"Error in cleanup task: {e}")
|
||||||
|
|
||||||
|
@tasks.loop(minutes=30)
|
||||||
|
async def stats_task():
|
||||||
|
"""Periodic statistics logging."""
|
||||||
|
try:
|
||||||
|
stats = await self.database.get_statistics()
|
||||||
|
self.logger.info(f"Database stats: {stats['total_users']} users, "
|
||||||
|
f"{stats['total_servers']} servers, "
|
||||||
|
f"{stats['database_size']} bytes")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.error(f"Error in stats task: {e}")
|
||||||
|
|
||||||
|
# Assign tasks to instance
|
||||||
|
self.cleanup_task = cleanup_task
|
||||||
|
self.stats_task = stats_task
|
||||||
|
|
||||||
async def on_ready(self):
|
async def on_ready(self):
|
||||||
"""Called when the client is ready."""
|
"""Called when the client is ready."""
|
||||||
self.logger.info(f"Logged in as {self.user} (ID: {self.user.id})")
|
self.logger.info(f"Logged in as {self.user} (ID: {self.user.id})")
|
||||||
self.logger.info(f"Connected to {len(self.guilds)} servers")
|
self.logger.info(f"Connected to {len(self.guilds)} servers")
|
||||||
|
|
||||||
|
# Start background tasks after we're ready
|
||||||
|
self.cleanup_task.start()
|
||||||
|
self.stats_task.start()
|
||||||
|
|
||||||
# Initial scan of server members
|
# Initial scan of server members
|
||||||
await self._scan_all_servers()
|
await self._scan_all_servers()
|
||||||
|
|
||||||
|
@ -180,6 +214,11 @@ async def _get_user_bio(self, user) -> Optional[str]:
|
||||||
try:
|
try:
|
||||||
bio = None
|
bio = None
|
||||||
|
|
||||||
|
# Debug logging
|
||||||
|
self.logger.debug(f"User object type: {type(user)}")
|
||||||
|
self.logger.debug(f"User attributes: {[attr for attr in dir(user) if not attr.startswith('_')]}")
|
||||||
|
self.logger.debug(f"Client methods: {[method for method in dir(self) if 'profile' in method.lower()]}")
|
||||||
|
|
||||||
# Method 1: Check if user object already has bio (for ClientUser)
|
# Method 1: Check if user object already has bio (for ClientUser)
|
||||||
if hasattr(user, 'bio') and user.bio:
|
if hasattr(user, 'bio') and user.bio:
|
||||||
bio = user.bio
|
bio = user.bio
|
||||||
|
@ -207,7 +246,6 @@ async def _get_user_bio(self, user) -> Optional[str]:
|
||||||
else:
|
else:
|
||||||
profile = fetched_user
|
profile = fetched_user
|
||||||
|
|
||||||
|
|
||||||
if profile:
|
if profile:
|
||||||
# Check all possible bio attributes
|
# Check all possible bio attributes
|
||||||
bio_attrs = ['bio', 'display_bio', 'about', 'about_me', 'description']
|
bio_attrs = ['bio', 'display_bio', 'about', 'about_me', 'description']
|
||||||
|
@ -326,33 +364,6 @@ async def _get_user_bio(self, user) -> Optional[str]:
|
||||||
self.logger.debug(f"Could not get activity for user {user.name}: {e}")
|
self.logger.debug(f"Could not get activity for user {user.name}: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@tasks.loop(hours=1)
|
|
||||||
async def cleanup_task(self):
|
|
||||||
"""Periodic cleanup task."""
|
|
||||||
try:
|
|
||||||
# Clean up old backups
|
|
||||||
await self.database.cleanup_old_backups()
|
|
||||||
|
|
||||||
# Clear processed users set to allow re-processing
|
|
||||||
self.processed_users.clear()
|
|
||||||
|
|
||||||
self.logger.info("Cleanup task completed")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f"Error in cleanup task: {e}")
|
|
||||||
|
|
||||||
@tasks.loop(minutes=30)
|
|
||||||
async def stats_task(self):
|
|
||||||
"""Periodic statistics logging."""
|
|
||||||
try:
|
|
||||||
stats = await self.database.get_statistics()
|
|
||||||
self.logger.info(f"Database stats: {stats['total_users']} users, "
|
|
||||||
f"{stats['total_servers']} servers, "
|
|
||||||
f"{stats['database_size']} bytes")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f"Error in stats task: {e}")
|
|
||||||
|
|
||||||
async def export_data(self, format_type: str = "csv", output_path: str = None):
|
async def export_data(self, format_type: str = "csv", output_path: str = None):
|
||||||
"""Export collected data."""
|
"""Export collected data."""
|
||||||
if output_path is None:
|
if output_path is None:
|
||||||
|
@ -376,8 +387,10 @@ async def _get_user_bio(self, user) -> Optional[str]:
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""Clean shutdown."""
|
"""Clean shutdown."""
|
||||||
# Cancel background tasks
|
# Cancel background tasks if they exist and are running
|
||||||
|
if hasattr(self, 'cleanup_task') and not self.cleanup_task.is_finished():
|
||||||
self.cleanup_task.cancel()
|
self.cleanup_task.cancel()
|
||||||
|
if hasattr(self, 'stats_task') and not self.stats_task.is_finished():
|
||||||
self.stats_task.cancel()
|
self.stats_task.cancel()
|
||||||
|
|
||||||
# Close parent client
|
# Close parent client
|
||||||
|
|
Loading…
Reference in a new issue