受欢迎的博客标签

NopCommerce 4.x (4)-robots.txt and sitemap.xml in NopCommerce

Published

Sitemaps XML format  https://www.sitemaps.org/protocol.html

1.robots.txt

step 1: Controller

\src\Presentation\Nop.Web\Controllers\CommonController.cs

 public virtual IActionResult RobotsTextFile()
        {
            var robotsFileContent = _commonModelFactory.PrepareRobotsTextFile();
            return Content(robotsFileContent, MimeTypes.TextPlain);
        }

src\Presentation\Nop.Web\Factories\CommonModelFactory.cs

if robots.custom.txt  exists return robots.custom.txt

if robots.custom.txt not  exists create and return robots.txt

if robots.custom.txt not  exists but robots.additions.txt exists,first create robots.txt ,then add robots.custom.txt and return robots.txt

 public virtual string PrepareRobotsTextFile()
        {
            var sb = new StringBuilder();

            //if robots.custom.txt exists, let's use it instead of hard-coded data below
            var robotsFilePath = _fileProvider.Combine(_fileProvider.MapPath("~/"), "robots.custom.txt");

            //if robots.custom.txt exists, let's use it instead of hard-coded data below
           // string robotsFilePath = System.IO.Path.Combine(CommonHelper.MapPath("~/"), "robots.custom.txt");
            if (System.IO.File.Exists(robotsFilePath))
            {
                //the robots.txt file exists
                string robotsFileContent = System.IO.File.ReadAllText(robotsFilePath);
                sb.Append(robotsFileContent);
            }
            else
            {
                //doesn't exist. Let's generate it (default behavior)

                var disallowPaths = new List<string>
                {
                    "/admin",
                    "/bin/",
                    "/files/",
                    "/files/exportimport/",
                    "/country/getstatesbycountryid",
                    "/install",
                    "/setproductreviewhelpfulness",
                };
                var localizableDisallowPaths = new List<string>
                {
                    "/addproducttocart/catalog/",
                    "/addproducttocart/details/",
                   
                    "/wishlist",
                };


                const string newLine = "\r\n"; //Environment.NewLine
                sb.Append("User-agent: *");
                sb.Append(newLine);

                //sitemaps
                if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
                {
                    //URLs are localizable. Append SEO code
                    foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id.ToString()))
                    {
                        sb.AppendFormat("Sitemap: {0}{1}/sitemap.xml", _storeContext.CurrentStore.Url, language.UniqueSeoCode);
                        sb.Append(newLine);
                    }
                }
                else
                {
                    //localizable paths (without SEO code)
                    sb.AppendFormat("Sitemap: {0}sitemap.xml", _storeContext.CurrentStore.Url);
                    sb.Append(newLine);
                }

                //usual paths
                foreach (var path in disallowPaths)
                {
                    sb.AppendFormat("Disallow: {0}", path);
                    sb.Append(newLine);
                }
                //localizable paths (without SEO code)
                foreach (var path in localizableDisallowPaths)
                {
                    sb.AppendFormat("Disallow: {0}", path);
                    sb.Append(newLine);
                }
                if (_localizationSettings.SeoFriendlyUrlsForLanguagesEnabled)
                {
                    //URLs are localizable. Append SEO code
                    foreach (var language in _languageService.GetAllLanguages(storeId: _storeContext.CurrentStore.Id.ToString()))
                    {
                        foreach (var path in localizableDisallowPaths)
                        {
                            sb.AppendFormat("Disallow: /{0}{1}", language.UniqueSeoCode, path);
                            sb.Append(newLine);
                        }
                    }
                }

                //load and add robots.txt additions to the end of file.
                string robotsAdditionsFile = System.IO.Path.Combine(CommonHelper.MapPath("~/"), "robots.additions.txt");
                if (System.IO.File.Exists(robotsAdditionsFile))
                {
                    string robotsFileContent = System.IO.File.ReadAllText(robotsAdditionsFile);
                    sb.Append(robotsFileContent);
                }
            }

            return sb.ToString();
        }

http://localhost:6999/common/RobotsTextFile

User-agent: *
Allow: /
User-agent: MJ12bot
Disallow: /

step 2: register router

 

http://localhost:6999/Robots.txt

User-agent: *
Allow: /

 

2.sitemap.xml

step

1.get all url come from product,blog etc. to list<url>

2.convert list<url> to   var sitemapUrls = new List<SitemapUrl>();

3. write sitemap.xml  come from List<SitemapUrl>()

4.CommonController's action  SitemapXml() output sitemap.xml as “Text/XML”

5.register router,which convert CommonController ,action  SitemapXml() to host://sitemap.xml

 

\src\Presentation\Nop.Web\Controllers\CommonController.cs

  [CheckAccessClosedStore(true)]
        public virtual IActionResult SitemapXml(int? id)
        {
            if (!_commonSettings.SitemapEnabled)
                return Content("_commonSettings.SitemapEnabled is Disable");
           // return RedirectToRoute("HomePage");

             var siteMap = _commonModelFactory.PrepareSitemapXml(this.Url, id);
            return Content(siteMap, "text/xml");
        }

\src\Presentation\Nop.Web\Factories\CommonModelFactory.cs

 public virtual string PrepareSitemapXml(IUrlHelper url, int? id)
        {
            string cacheKey = string.Format(ModelCacheEventConsumer.SITEMAP_SEO_MODEL_KEY, id,
                _workContext.WorkingLanguage.Id,
                string.Join(",", _workContext.CurrentCustomer.GetCustomerRoleIds()),
                _storeContext.CurrentStore.Id);
            var siteMap = _cacheManager.Get(cacheKey, () => _sitemapGenerator.Generate(url, id));
            return siteMap;
        }

This will build an XML sitemap for better index with search engines.page or not.

  /// <summary>
        /// This will build an XML sitemap for better index with search engines.
        /// See http://en.wikipedia.org/wiki/Sitemaps for more information.
        /// </summary>
        /// <param name="urlHelper">URL helper</param>
        /// <param name="id">Sitemap identifier</param>
        /// <returns>Sitemap.xml as string</returns>
        public virtual string Generate(IUrlHelper urlHelper, int? id)
        {
            using (var stream = new MemoryStream())
            {
                Generate(urlHelper, stream, id);
                return Encoding.UTF8.GetString(stream.ToArray());
            }
        }

        /// <summary>
        /// This will build an XML sitemap for better index with search engines.
        /// See http://en.wikipedia.org/wiki/Sitemaps for more information.
        /// </summary>
        /// <param name="urlHelper">URL helper</param>
        /// <param name="id">Sitemap identifier</param>
        /// <param name="stream">Stream of sitemap.</param>
        public virtual void Generate(IUrlHelper urlHelper, Stream stream, int? id)
        {
            //generate all URLs for the sitemap
            var sitemapUrls = GenerateUrls(urlHelper);

            //split URLs into separate lists based on the max size 
            var sitemaps = sitemapUrls.Select((url, index) => new { Index = index, Value = url })
                .GroupBy(group => group.Index / maxSitemapUrlNumber).Select(group => group.Select(url => url.Value).ToList()).ToList();

            if (!sitemaps.Any())
                return;

            if (id.HasValue)
            {
                //requested sitemap does not exist
                if (id.Value == 0 || id.Value > sitemaps.Count)
                    return;

                //otherwise write a certain numbered sitemap file into the stream
                WriteSitemap(urlHelper, stream, sitemaps.ElementAt(id.Value - 1));

            }
            else
            {
                //URLs more than the maximum allowable, so generate a sitemap index file
                if (sitemapUrls.Count >= maxSitemapUrlNumber)
                {
                    //write a sitemap index file into the stream
                    WriteSitemapIndex(urlHelper, stream, sitemaps.Count);
                }
                else
                {
                    //otherwise generate a standard sitemap
                    WriteSitemap(urlHelper, stream, sitemaps.First());
                }
            }
        }

 

Add all url to IList<SitemapUrl>

 protected virtual IList<SitemapUrl> GenerateUrls(IUrlHelper urlHelper)
        {
            var sitemapUrls = new List<SitemapUrl>();

            //home page
            var homePageUrl = urlHelper.RouteUrl("HomePage", null, GetHttpProtocol());
            sitemapUrls.Add(new SitemapUrl(homePageUrl, UpdateFrequency.Weekly, DateTime.UtcNow));

            //search products
            var productSearchUrl = urlHelper.RouteUrl("ProductSearch", null, GetHttpProtocol());
            sitemapUrls.Add(new SitemapUrl(productSearchUrl, UpdateFrequency.Weekly, DateTime.UtcNow));

            //contact us
            var contactUsUrl = urlHelper.RouteUrl("ContactUs", null, GetHttpProtocol());
            sitemapUrls.Add(new SitemapUrl(contactUsUrl, UpdateFrequency.Weekly, DateTime.UtcNow));

 //topics
            sitemapUrls.AddRange(GetTopicUrls(urlHelper));

            //blog posts
            sitemapUrls.AddRange( GetBlogPostsUrls(urlHelper).Result);

          ...

            return sitemapUrls;
        }

 

 

step 2: register router

\src\Presentation\Nop.Web\Infrastructure\RouteProvider.cs

 public static class RegisterRoutes
    {

        /// <summary>
        /// static router table
        /// </summary>
        /// <param name="routes"></param>
        public static void MapModuleRoute(this IRouteBuilder routes)
        {
           //sitemap (XML)
            routes.MapRoute("sitemap.xml", "sitemap.xml",
                new { controller = "Common", action = "SitemapXml" });

            routes.MapRoute("sitemap-indexed.xml", "sitemap-{Id:min(0)}.xml",
                new { controller = "Common", action = "SitemapXml" });
         
             //robots.txt
            routes.MapRoute("robots.txt", "robots.txt",
                new { controller = "Common", action = "RobotsTextFile" });
        }
    }

 

public void Configure(IApplicationBuilder app, IHostingEnvironment env, IApplicationLifetime appLifetime, ILoggerFactory loggerFactory)
        {
          routes.MapModuleRoute();

        }