From 3df1663e4f7a3958e1c4016f57ad387d19879c90 Mon Sep 17 00:00:00 2001
From: Kevin Hu <kevinhu.sh@gmail.com>
Date: Tue, 26 Nov 2024 09:34:34 +0800
Subject: [PATCH] For security. (#3642)

### What problem does this PR solve?


### Type of change

- [x] Performance Improvement
---
 agent/component/crawler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/agent/component/crawler.py b/agent/component/crawler.py
index f0012d71..151942b6 100644
--- a/agent/component/crawler.py
+++ b/agent/component/crawler.py
@@ -17,6 +17,7 @@ from abc import ABC
 import asyncio
 from crawl4ai import AsyncWebCrawler
 from agent.component.base import ComponentBase, ComponentParamBase
+from api.utils.web_utils import is_valid_url
 
 
 class CrawlerParam(ComponentParamBase):
@@ -39,7 +40,7 @@ class Crawler(ComponentBase, ABC):
     def _run(self, history, **kwargs):
         ans = self.get_input()
         ans = " - ".join(ans["content"]) if "content" in ans else ""
-        if not ans:
+        if not is_valid_url(ans):
             return Crawler.be_output("")
         try:
             result = asyncio.run(self.get_web(ans))