Merge pull request #1821 from 0xi4o/bug/web-scrapper-ui

Fix: Error Handling in Web Scraper
FlowiseAI · Feb 27, 2024 · bdb93f7 · bdb93f7
2 parents 2d9bf58 + 07ce50c
commit bdb93f7
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 12 deletions.
diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
@@ -1245,18 +1245,22 @@ export class App {
  // ----------------------------------------
 
  this.app.get('/api/v1/fetch-links', async (req: Request, res: Response) => {
- const url = decodeURIComponent(req.query.url as string)
- const relativeLinksMethod = req.query.relativeLinksMethod as string
- if (!relativeLinksMethod) {
- return res.status(500).send('Please choose a Relative Links Method in Additional Parameters.')
- }
+ try {
+ const url = decodeURIComponent(req.query.url as string)
+ const relativeLinksMethod = req.query.relativeLinksMethod as string
+ if (!relativeLinksMethod) {
+ return res.status(500).send('Please choose a Relative Links Method in Additional Parameters.')
+ }
 
- const limit = parseInt(req.query.limit as string)
- if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
- const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)
- if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
+  const limit = parseInt(req.query.limit as string)
+  if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`)
+  const links: string[] = relativeLinksMethod === 'webCrawl' ? await webCrawl(url, limit) : await xmlScrape(url, limit)
+  if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`)
 
- res.json({ status: 'OK', links })
+ res.json({ status: 'OK', links })
+ } catch (e: any) {
+ return res.status(500).send('Could not fetch links from the URL.')
+ }
  })
 
  // ----------------------------------------

diff --git a/packages/ui/src/ui-component/dialog/ManageScrapedLinksDialog.js b/packages/ui/src/ui-component/dialog/ManageScrapedLinksDialog.js
@@ -16,7 +16,7 @@ import {
  Stack,
  Typography
 } from '@mui/material'
-import { IconTrash, IconX } from '@tabler/icons'
+import { IconEraser, IconTrash, IconX } from '@tabler/icons'
 import PerfectScrollbar from 'react-perfect-scrollbar'
 
 import { BackdropLoader } from 'ui-component/loading/BackdropLoader'
@@ -113,6 +113,10 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
  setSelectedLinks(links)
  }
 
+ const handleRemoveAllLinks = () => {
+ setSelectedLinks([])
+ }
+
  const handleSaveLinks = () => {
  onSave(url, selectedLinks)
  }
@@ -145,6 +149,7 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
  />
  </FormControl>
  <Button
+ disabled={!url}
  sx={{ borderRadius: '12px', mt: 1, display: 'flex', flexShrink: 0 }}
  size='small'
  variant='contained'
@@ -154,7 +159,21 @@ const ManageScrapedLinksDialog = ({ show, dialogProps, onCancel, onSave }) => {
  </Button>
  </Stack>
  </Box>
- <Typography sx={{ mb: 2, fontWeight: 500 }}>Scraped Links</Typography>
+ <Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', mb: 1.5 }}>
+ <Typography sx={{ fontWeight: 500 }}>Scraped Links</Typography>
+ {selectedLinks.length > 0 ? (
+ <StyledButton
+ sx={{ height: 'max-content', width: 'max-content' }}
+ variant='outlined'
+ color='error'
+ title='Clear All Links'
+ onClick={handleRemoveAllLinks}
+ startIcon={<IconEraser />}
+ >
+ Clear All
+ </StyledButton>
+ ) : null}
+ </Box>
  <>
  {loading && <BackdropLoader open={loading} />}
  {selectedLinks.length > 0 ? (