# robots.txt for Magento 1.9 (hicart.com)
# Optimized to reduce bot-driven CPU load

## ============================================================
## ALL CRAWLERS
## ============================================================
User-agent: *

## Crawl-delay: seconds between requests. ENABLED to throttle
## bots that overload the server. NOTE: Googlebot IGNORES this —
## set Google's rate in Search Console → Settings → Crawl rate.
## Bing, Yandex, and most others DO obey it.
Crawl-delay: 20

## Magento sitemap
Sitemap: https://www.hicart.com/media/sitemap/sitemap.xml

## ------------------------------------------------------------
## DEVELOPMENT / DUMP FILES
## ------------------------------------------------------------
Disallow: /CVS
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$

## ------------------------------------------------------------
## MAGENTO TECHNICAL FOLDERS
## ------------------------------------------------------------
Disallow: /admin/
Disallow: /app/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /dev/
Disallow: /api/
Disallow: /pkginfo/
Disallow: /shell/
Disallow: /var/
Disallow: /media/downloadable/
Disallow: /report/

## ------------------------------------------------------------
## MAGENTO TECHNICAL FILES
## ------------------------------------------------------------
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt
Disallow: /crons.txt

## ------------------------------------------------------------
## CRAWL TRAPS — layered navigation, sorting, pagination.
## THIS IS THE #1 CPU KILLER IN MAGENTO 1.9. These params
## multiply into millions of duplicate URLs. Block them all.
## ------------------------------------------------------------
Disallow: /*?dir=
Disallow: /*&dir=
Disallow: /*?mode=
Disallow: /*&mode=
Disallow: /*?limit=
Disallow: /*&limit=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*?price=
Disallow: /*&price=
Disallow: /*?p=
Disallow: /*&p=
Disallow: /*?product_list_mode=
Disallow: /*&product_list_mode=
Disallow: /*?product_list_limit=
Disallow: /*&product_list_limit=
Disallow: /*?product_list_order=
Disallow: /*&product_list_order=
Disallow: /*?cat=
Disallow: /*?manufacturer=
Disallow: /*?color=
Disallow: /*?size=

## Session IDs
Disallow: /*?SID=
Disallow: /*SID=

## ------------------------------------------------------------
## CHECKOUT / ACCOUNT / NON-INDEX PAGES (heavy, no SEO value)
## ------------------------------------------------------------
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /tag/
Disallow: /rss/

## Uncomment ONLY if your SEO URLs are active and index.php/
## duplicates are being crawled:
# Disallow: /index.php/

## ------------------------------------------------------------
## SERVER TECHNICAL FILES
## ------------------------------------------------------------
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php
Disallow: /test.php