Plaster

commonlisp
CL-USER> (defun crawl-for-files (url download-dir) (unless (cl-fad:directory-exists-p download-dir) (error 'simple-error :message "Folder not found: ~a" download-dir)) (multiple-value-bind (doc status properties) (drakma:http-request url) (when (= status 200) (let ((content-type (cl-ppcre:scan-to-strings "\\w+/\\w+" (cdr (assoc :content-type properties))))) (cond ((string= "text/html" content-type) (let* ((parsed (plump:parse doc)) (doc (lquery:$ parsed))) (for:for ((new-url across (lquery:$ doc "a" (attr "href")))) (when (cl-ppcre:scan "^[\\w_\\-]+.*$" new-url) (crawl-for-files (concatenate 'string url new-url) download-dir))))) ((string= "text/plain" content-type) (let ((filename (cl-ppcre:scan-to-strings "[^/?#]+\\.txt$" url))) (when filename (with-open-file (output (cl-fad:merge-pathnames-as-file download-dir filename) :direction :output :if-exists :supersede) (format output "~a" doc)) (format T "Saved ~a~%" filename)))))))))